{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Use TD3 to Play Pendulum-v0\n",
    "\n",
    "TensorFlow version"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "%matplotlib inline\n",
    "\n",
    "import sys\n",
    "import logging\n",
    "import itertools\n",
    "\n",
    "import numpy as np\n",
    "np.random.seed(0)\n",
    "import pandas as pd\n",
    "import gym\n",
    "import matplotlib.pyplot as plt\n",
    "import tensorflow.compat.v2 as tf\n",
    "tf.random.set_seed(0)\n",
    "from tensorflow import keras\n",
    "from tensorflow import nn\n",
    "from tensorflow import optimizers\n",
    "from tensorflow import losses\n",
    "from tensorflow.keras import layers\n",
    "from tensorflow.keras import models\n",
    "\n",
    "logging.basicConfig(level=logging.DEBUG,\n",
    "        format='%(asctime)s [%(levelname)s] %(message)s',\n",
    "        stream=sys.stdout, datefmt='%H:%M:%S')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:50 [INFO] env: <PendulumEnv<Pendulum-v0>>\n",
      "22:22:50 [INFO] action_space: Box(-2.0, 2.0, (1,), float32)\n",
      "22:22:50 [INFO] observation_space: Box(-8.0, 8.0, (3,), float32)\n",
      "22:22:50 [INFO] reward_range: (-inf, inf)\n",
      "22:22:50 [INFO] metadata: {'render.modes': ['human', 'rgb_array'], 'video.frames_per_second': 30}\n",
      "22:22:50 [INFO] _max_episode_steps: 200\n",
      "22:22:50 [INFO] _elapsed_steps: None\n",
      "22:22:50 [INFO] id: Pendulum-v0\n",
      "22:22:50 [INFO] entry_point: gym.envs.classic_control:PendulumEnv\n",
      "22:22:50 [INFO] reward_threshold: None\n",
      "22:22:50 [INFO] nondeterministic: False\n",
      "22:22:50 [INFO] max_episode_steps: 200\n",
      "22:22:50 [INFO] _kwargs: {}\n",
      "22:22:50 [INFO] _env_name: Pendulum\n"
     ]
    }
   ],
   "source": [
    "env = gym.make('Pendulum-v0')\n",
    "env.seed(0)\n",
    "for key in vars(env):\n",
    "    logging.info('%s: %s', key, vars(env)[key])\n",
    "for key in vars(env.spec):\n",
    "    logging.info('%s: %s', key, vars(env.spec)[key])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "class DQNReplayer:\n",
    "    def __init__(self, capacity):\n",
    "        self.memory = pd.DataFrame(index=range(capacity),\n",
    "                columns=['state', 'action', 'reward', 'next_state', 'done'])\n",
    "        self.i = 0\n",
    "        self.count = 0\n",
    "        self.capacity = capacity\n",
    "\n",
    "    def store(self, *args):\n",
    "        self.memory.loc[self.i] = args\n",
    "        self.i = (self.i + 1) % self.capacity\n",
    "        self.count = min(self.count + 1, self.capacity)\n",
    "\n",
    "    def sample(self, size):\n",
    "        indices = np.random.choice(self.count, size=size)\n",
    "        return (np.stack(self.memory.loc[indices, field]) for field in\n",
    "                self.memory.columns)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "class OrnsteinUhlenbeckProcess:\n",
    "    def __init__(self, x0):\n",
    "        self.x = x0\n",
    "\n",
    "    def __call__(self, mu=0., sigma=1., theta=.15, dt=.01):\n",
    "        n = np.random.normal(size=self.x.shape)\n",
    "        self.x += (theta * (mu - self.x) * dt + sigma * np.sqrt(dt) * n)\n",
    "        return self.x"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "class TD3Agent:\n",
    "    def __init__(self, env):\n",
    "        state_dim = env.observation_space.shape[0]\n",
    "        self.action_dim = env.action_space.shape[0]\n",
    "        self.action_low = env.action_space.low\n",
    "        self.action_high = env.action_space.high\n",
    "        self.gamma = 0.99\n",
    "\n",
    "        self.replayer = DQNReplayer(20000)\n",
    "\n",
    "        self.actor_evaluate_net = self.build_net(\n",
    "                input_size=state_dim, hidden_sizes=[32, 64],\n",
    "                output_size=self.action_dim, output_activation=nn.tanh)\n",
    "        self.actor_target_net = models.clone_model(self.actor_evaluate_net)\n",
    "        self.actor_target_net.set_weights(self.actor_evaluate_net.get_weights())\n",
    "\n",
    "        self.critic0_evaluate_net = self.build_net(\n",
    "                input_size=state_dim+self.action_dim, hidden_sizes=[64, 128])\n",
    "        self.critic0_target_net = models.clone_model(self.critic0_evaluate_net)\n",
    "        self.critic0_target_net.set_weights(self.critic0_evaluate_net.get_weights())\n",
    "\n",
    "        self.critic1_evaluate_net = self.build_net(\n",
    "                input_size=state_dim+self.action_dim, hidden_sizes=[64, 128])\n",
    "        self.critic1_target_net = models.clone_model(self.critic1_evaluate_net)\n",
    "        self.critic1_target_net.set_weights(self.critic1_evaluate_net.get_weights())\n",
    "\n",
    "    def build_net(self, input_size=None, hidden_sizes=None, output_size=1,\n",
    "                activation=nn.relu, output_activation=None,\n",
    "                loss=losses.mse, learning_rate=0.001):\n",
    "        model = keras.Sequential()\n",
    "        for layer, hidden_size in enumerate(hidden_sizes):\n",
    "            kwargs = {'input_shape' : (input_size,)} if layer == 0 else {}\n",
    "            model.add(layers.Dense(units=hidden_size,\n",
    "                    activation=activation, **kwargs))\n",
    "        model.add(layers.Dense(units=output_size,\n",
    "                activation=output_activation))\n",
    "        optimizer = optimizers.Adam(learning_rate)\n",
    "        model.compile(optimizer=optimizer, loss=loss)\n",
    "        return model\n",
    "\n",
    "    def reset(self, mode=None):\n",
    "        self.mode = mode\n",
    "        if self.mode == 'train':\n",
    "            self.trajectory = []\n",
    "            self.noise = OrnsteinUhlenbeckProcess(np.zeros((self.action_dim,)))\n",
    "\n",
    "    def step(self, observation, reward, done):\n",
    "        if self.mode == 'train' and self.replayer.count < 3000:\n",
    "            action = np.random.uniform(self.action_low, self.action_high)\n",
    "        else:\n",
    "            action = self.actor_evaluate_net.predict(observation[np.newaxis])[0]\n",
    "        if self.mode == 'train':\n",
    "            noise = self.noise(sigma=0.1)\n",
    "            action = (action + noise).clip(self.action_low, self.action_high)\n",
    "\n",
    "            self.trajectory += [observation, reward, done, action]\n",
    "            if len(self.trajectory) >= 8:\n",
    "                state, _, _, act, next_state, reward, done, _ = self.trajectory[-8:]\n",
    "                self.replayer.store(state, act, reward, next_state, done)\n",
    "\n",
    "            if self.replayer.count >= 3000:\n",
    "                self.learn()\n",
    "        return action\n",
    "\n",
    "    def close(self):\n",
    "        pass\n",
    "\n",
    "    def update_net(self, target_net, evaluate_net, learning_rate=0.005):\n",
    "        average_weights = [(1. - learning_rate) * t + learning_rate * e for t, e\n",
    "                in zip(target_net.get_weights(), evaluate_net.get_weights())]\n",
    "        target_net.set_weights(average_weights)\n",
    "\n",
    "    def learn(self):\n",
    "        # replay\n",
    "        states, actions, rewards, next_states, dones = self.replayer.sample(64)\n",
    "        state_tensor = tf.convert_to_tensor(states, dtype=tf.float32)\n",
    "\n",
    "        # learn critic\n",
    "        next_actions = self.actor_target_net.predict(next_states)\n",
    "        next_noises = np.random.normal(0, 0.2, size=next_actions.shape)\n",
    "        next_actions = (next_actions + next_noises).clip(self.action_low, self.action_high)\n",
    "        state_actions = np.hstack([states, actions])\n",
    "        next_state_actions = np.hstack([next_states, next_actions])\n",
    "        next_q0s = self.critic0_target_net.predict(next_state_actions)[:, 0]\n",
    "        next_q1s = self.critic1_target_net.predict(next_state_actions)[:, 0]\n",
    "        next_qs = np.minimum(next_q0s, next_q1s)\n",
    "        targets = rewards + (1. - dones) * self.gamma * next_qs\n",
    "        self.critic0_evaluate_net.fit(state_actions, targets[:, np.newaxis], verbose=0)\n",
    "        self.critic1_evaluate_net.fit(state_actions, targets[:, np.newaxis], verbose=0)\n",
    "\n",
    "        # learn actor\n",
    "        with tf.GradientTape() as tape:\n",
    "            action_tensor = self.actor_evaluate_net(state_tensor)\n",
    "            state_action_tensor = tf.concat([state_tensor, action_tensor], axis=1)\n",
    "            q_tensor = self.critic0_evaluate_net(state_action_tensor)\n",
    "            loss_tensor = -tf.reduce_mean(q_tensor)\n",
    "        grad_tensors = tape.gradient(loss_tensor, self.actor_evaluate_net.variables)\n",
    "        self.actor_evaluate_net.optimizer.apply_gradients(zip(\n",
    "                grad_tensors, self.actor_evaluate_net.variables))\n",
    "\n",
    "        self.update_net(self.critic0_target_net, self.critic0_evaluate_net)\n",
    "        self.update_net(self.critic1_target_net, self.critic1_evaluate_net)\n",
    "        self.update_net(self.actor_target_net, self.actor_evaluate_net)\n",
    "\n",
    "\n",
    "agent = TD3Agent(env)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22:22:51 [INFO] ==== train ====\n",
      "22:22:51 [DEBUG] train episode 0: reward = -1744.13, steps = 200\n",
      "22:22:51 [DEBUG] train episode 1: reward = -1025.25, steps = 200\n",
      "22:22:52 [DEBUG] train episode 2: reward = -1590.20, steps = 200\n",
      "22:22:52 [DEBUG] train episode 3: reward = -1137.77, steps = 200\n",
      "22:22:52 [DEBUG] train episode 4: reward = -1675.82, steps = 200\n",
      "22:22:52 [DEBUG] train episode 5: reward = -1632.97, steps = 200\n",
      "22:22:52 [DEBUG] train episode 6: reward = -753.85, steps = 200\n",
      "22:22:52 [DEBUG] train episode 7: reward = -1833.66, steps = 200\n",
      "22:22:52 [DEBUG] train episode 8: reward = -936.49, steps = 200\n",
      "22:22:52 [DEBUG] train episode 9: reward = -1622.68, steps = 200\n",
      "22:22:52 [DEBUG] train episode 10: reward = -1307.43, steps = 200\n",
      "22:22:52 [DEBUG] train episode 11: reward = -908.99, steps = 200\n",
      "22:22:52 [DEBUG] train episode 12: reward = -1504.19, steps = 200\n",
      "22:22:53 [DEBUG] train episode 13: reward = -1003.41, steps = 200\n",
      "22:22:53 [DEBUG] train episode 14: reward = -921.67, steps = 200\n",
      "22:24:27 [DEBUG] train episode 15: reward = -1156.67, steps = 200\n",
      "22:26:07 [DEBUG] train episode 16: reward = -1467.30, steps = 200\n",
      "22:27:30 [DEBUG] train episode 17: reward = -1550.39, steps = 200\n",
      "22:28:55 [DEBUG] train episode 18: reward = -1696.63, steps = 200\n",
      "22:30:14 [DEBUG] train episode 19: reward = -1086.24, steps = 200\n",
      "22:31:49 [DEBUG] train episode 20: reward = -1313.67, steps = 200\n",
      "22:33:18 [DEBUG] train episode 21: reward = -1413.73, steps = 200\n",
      "22:34:43 [DEBUG] train episode 22: reward = -1518.39, steps = 200\n",
      "22:36:18 [DEBUG] train episode 23: reward = -1183.81, steps = 200\n",
      "22:38:12 [DEBUG] train episode 24: reward = -991.98, steps = 200\n",
      "22:40:13 [DEBUG] train episode 25: reward = -1113.87, steps = 200\n",
      "22:42:31 [DEBUG] train episode 26: reward = -1015.62, steps = 200\n",
      "22:44:54 [DEBUG] train episode 27: reward = -752.92, steps = 200\n",
      "22:47:12 [DEBUG] train episode 28: reward = -625.19, steps = 200\n",
      "22:49:13 [DEBUG] train episode 29: reward = -991.54, steps = 200\n",
      "22:51:29 [DEBUG] train episode 30: reward = -376.51, steps = 200\n",
      "22:53:38 [DEBUG] train episode 31: reward = -497.96, steps = 200\n",
      "22:55:53 [DEBUG] train episode 32: reward = -126.30, steps = 200\n",
      "22:58:03 [DEBUG] train episode 33: reward = -242.28, steps = 200\n",
      "23:00:18 [DEBUG] train episode 34: reward = -481.37, steps = 200\n",
      "23:02:46 [DEBUG] train episode 35: reward = -497.32, steps = 200\n",
      "23:05:10 [DEBUG] train episode 36: reward = -606.38, steps = 200\n",
      "23:07:26 [DEBUG] train episode 37: reward = -126.52, steps = 200\n",
      "23:09:43 [DEBUG] train episode 38: reward = -126.26, steps = 200\n",
      "23:12:19 [DEBUG] train episode 39: reward = -241.38, steps = 200\n",
      "23:15:23 [DEBUG] train episode 40: reward = -652.64, steps = 200\n",
      "23:18:22 [DEBUG] train episode 41: reward = -790.80, steps = 200\n",
      "23:21:21 [DEBUG] train episode 42: reward = -250.58, steps = 200\n",
      "23:24:24 [DEBUG] train episode 43: reward = -1.79, steps = 200\n",
      "23:27:23 [DEBUG] train episode 44: reward = -254.04, steps = 200\n",
      "23:30:22 [DEBUG] train episode 45: reward = -121.56, steps = 200\n",
      "23:33:22 [DEBUG] train episode 46: reward = -239.31, steps = 200\n",
      "23:36:20 [DEBUG] train episode 47: reward = -120.61, steps = 200\n",
      "23:39:24 [DEBUG] train episode 48: reward = -126.17, steps = 200\n",
      "23:42:28 [DEBUG] train episode 49: reward = -125.57, steps = 200\n",
      "23:45:45 [DEBUG] train episode 50: reward = -484.86, steps = 200\n",
      "23:49:07 [DEBUG] train episode 51: reward = -122.38, steps = 200\n",
      "23:52:26 [DEBUG] train episode 52: reward = -238.84, steps = 200\n",
      "23:55:33 [DEBUG] train episode 53: reward = -245.61, steps = 200\n",
      "23:58:32 [DEBUG] train episode 54: reward = -247.65, steps = 200\n",
      "00:01:29 [DEBUG] train episode 55: reward = -486.61, steps = 200\n",
      "00:04:27 [DEBUG] train episode 56: reward = -125.16, steps = 200\n",
      "00:07:29 [DEBUG] train episode 57: reward = -243.55, steps = 200\n",
      "00:10:27 [DEBUG] train episode 58: reward = -119.64, steps = 200\n",
      "00:13:24 [DEBUG] train episode 59: reward = -237.30, steps = 200\n",
      "00:16:22 [DEBUG] train episode 60: reward = -123.70, steps = 200\n",
      "00:19:16 [DEBUG] train episode 61: reward = -240.52, steps = 200\n",
      "00:22:15 [DEBUG] train episode 62: reward = -238.56, steps = 200\n",
      "00:25:12 [DEBUG] train episode 63: reward = -121.25, steps = 200\n",
      "00:28:12 [DEBUG] train episode 64: reward = -240.12, steps = 200\n",
      "00:31:09 [DEBUG] train episode 65: reward = -122.46, steps = 200\n",
      "00:34:13 [DEBUG] train episode 66: reward = -125.17, steps = 200\n",
      "00:37:11 [DEBUG] train episode 67: reward = -365.42, steps = 200\n",
      "00:40:07 [DEBUG] train episode 68: reward = -475.81, steps = 200\n",
      "00:43:04 [DEBUG] train episode 69: reward = -239.04, steps = 200\n",
      "00:45:58 [DEBUG] train episode 70: reward = -120.79, steps = 200\n",
      "00:48:54 [DEBUG] train episode 71: reward = -124.61, steps = 200\n",
      "00:51:51 [DEBUG] train episode 72: reward = -468.28, steps = 200\n",
      "00:54:47 [DEBUG] train episode 73: reward = -126.16, steps = 200\n",
      "00:57:43 [DEBUG] train episode 74: reward = -234.84, steps = 200\n",
      "01:00:44 [DEBUG] train episode 75: reward = -125.65, steps = 200\n",
      "01:03:40 [DEBUG] train episode 76: reward = -357.30, steps = 200\n",
      "01:06:37 [DEBUG] train episode 77: reward = -123.65, steps = 200\n",
      "01:09:35 [DEBUG] train episode 78: reward = -120.43, steps = 200\n",
      "01:12:31 [DEBUG] train episode 79: reward = -124.02, steps = 200\n",
      "01:15:26 [DEBUG] train episode 80: reward = -505.90, steps = 200\n",
      "01:18:22 [DEBUG] train episode 81: reward = -126.72, steps = 200\n",
      "01:21:17 [DEBUG] train episode 82: reward = -596.69, steps = 200\n",
      "01:24:13 [DEBUG] train episode 83: reward = -124.80, steps = 200\n",
      "01:27:11 [DEBUG] train episode 84: reward = -121.46, steps = 200\n",
      "01:30:06 [DEBUG] train episode 85: reward = -121.98, steps = 200\n",
      "01:33:04 [DEBUG] train episode 86: reward = -593.72, steps = 200\n",
      "01:36:00 [DEBUG] train episode 87: reward = -240.93, steps = 200\n",
      "01:38:58 [DEBUG] train episode 88: reward = -247.89, steps = 200\n",
      "01:41:53 [DEBUG] train episode 89: reward = -121.46, steps = 200\n",
      "01:44:50 [DEBUG] train episode 90: reward = -123.04, steps = 200\n",
      "01:47:46 [DEBUG] train episode 91: reward = -120.72, steps = 200\n",
      "01:50:42 [DEBUG] train episode 92: reward = -236.45, steps = 200\n",
      "01:53:39 [DEBUG] train episode 93: reward = -125.30, steps = 200\n",
      "01:56:35 [DEBUG] train episode 94: reward = -126.42, steps = 200\n",
      "01:59:30 [DEBUG] train episode 95: reward = -506.16, steps = 200\n",
      "02:02:25 [DEBUG] train episode 96: reward = -122.54, steps = 200\n",
      "02:05:20 [DEBUG] train episode 97: reward = -360.87, steps = 200\n",
      "02:08:19 [DEBUG] train episode 98: reward = -121.46, steps = 200\n",
      "02:11:16 [DEBUG] train episode 99: reward = -471.62, steps = 200\n",
      "02:14:13 [DEBUG] train episode 100: reward = -363.90, steps = 200\n",
      "02:17:09 [DEBUG] train episode 101: reward = -238.32, steps = 200\n",
      "02:20:04 [DEBUG] train episode 102: reward = -598.70, steps = 200\n",
      "02:23:01 [DEBUG] train episode 103: reward = -494.53, steps = 200\n",
      "02:25:58 [DEBUG] train episode 104: reward = -492.42, steps = 200\n",
      "02:28:58 [DEBUG] train episode 105: reward = -124.28, steps = 200\n",
      "02:31:52 [DEBUG] train episode 106: reward = -244.48, steps = 200\n",
      "02:34:48 [DEBUG] train episode 107: reward = -356.00, steps = 200\n",
      "02:37:44 [DEBUG] train episode 108: reward = -119.03, steps = 200\n",
      "02:40:40 [DEBUG] train episode 109: reward = -489.83, steps = 200\n",
      "02:43:39 [DEBUG] train episode 110: reward = -124.21, steps = 200\n",
      "02:46:35 [DEBUG] train episode 111: reward = -125.19, steps = 200\n",
      "02:49:30 [DEBUG] train episode 112: reward = -354.99, steps = 200\n",
      "02:52:27 [DEBUG] train episode 113: reward = -121.10, steps = 200\n",
      "02:55:23 [DEBUG] train episode 114: reward = -119.28, steps = 200\n",
      "02:58:05 [DEBUG] train episode 115: reward = -468.05, steps = 200\n",
      "03:00:41 [DEBUG] train episode 116: reward = -367.44, steps = 200\n",
      "03:03:16 [DEBUG] train episode 117: reward = -124.05, steps = 200\n",
      "03:05:51 [DEBUG] train episode 118: reward = -237.68, steps = 200\n",
      "03:08:28 [DEBUG] train episode 119: reward = -123.57, steps = 200\n",
      "03:11:02 [DEBUG] train episode 120: reward = -363.97, steps = 200\n",
      "03:13:37 [DEBUG] train episode 121: reward = -120.68, steps = 200\n",
      "03:16:13 [DEBUG] train episode 122: reward = -365.84, steps = 200\n",
      "03:18:47 [DEBUG] train episode 123: reward = -497.26, steps = 200\n",
      "03:21:18 [DEBUG] train episode 124: reward = -123.05, steps = 200\n",
      "03:23:51 [DEBUG] train episode 125: reward = -483.80, steps = 200\n",
      "03:26:23 [DEBUG] train episode 126: reward = -125.38, steps = 200\n",
      "03:28:57 [DEBUG] train episode 127: reward = -125.54, steps = 200\n",
      "03:31:28 [DEBUG] train episode 128: reward = -121.31, steps = 200\n",
      "03:33:58 [DEBUG] train episode 129: reward = -349.95, steps = 200\n",
      "03:36:27 [DEBUG] train episode 130: reward = -1.79, steps = 200\n",
      "03:38:59 [DEBUG] train episode 131: reward = -360.27, steps = 200\n",
      "03:41:31 [DEBUG] train episode 132: reward = -127.78, steps = 200\n",
      "03:44:02 [DEBUG] train episode 133: reward = -184.57, steps = 200\n",
      "03:46:33 [DEBUG] train episode 134: reward = -492.26, steps = 200\n",
      "03:49:04 [DEBUG] train episode 135: reward = -522.55, steps = 200\n",
      "03:51:36 [DEBUG] train episode 136: reward = -241.31, steps = 200\n",
      "03:54:08 [DEBUG] train episode 137: reward = -475.77, steps = 200\n",
      "03:56:38 [DEBUG] train episode 138: reward = -242.52, steps = 200\n",
      "03:59:10 [DEBUG] train episode 139: reward = -483.96, steps = 200\n",
      "04:01:42 [DEBUG] train episode 140: reward = -238.91, steps = 200\n",
      "04:04:13 [DEBUG] train episode 141: reward = -355.65, steps = 200\n",
      "04:06:43 [DEBUG] train episode 142: reward = -525.41, steps = 200\n",
      "04:09:16 [DEBUG] train episode 143: reward = -122.80, steps = 200\n",
      "04:11:57 [DEBUG] train episode 144: reward = -119.83, steps = 200\n",
      "04:14:27 [DEBUG] train episode 145: reward = -121.93, steps = 200\n",
      "04:16:57 [DEBUG] train episode 146: reward = -129.13, steps = 200\n",
      "04:19:23 [DEBUG] train episode 147: reward = -366.97, steps = 200\n",
      "04:21:37 [DEBUG] train episode 148: reward = -127.54, steps = 200\n",
      "04:23:50 [DEBUG] train episode 149: reward = -1677.82, steps = 200\n",
      "04:26:04 [DEBUG] train episode 150: reward = -1708.18, steps = 200\n",
      "04:28:19 [DEBUG] train episode 151: reward = -359.16, steps = 200\n",
      "04:30:33 [DEBUG] train episode 152: reward = -3.74, steps = 200\n",
      "04:32:38 [DEBUG] train episode 153: reward = -237.41, steps = 200\n",
      "04:34:35 [DEBUG] train episode 154: reward = -370.23, steps = 200\n",
      "04:36:34 [DEBUG] train episode 155: reward = -366.13, steps = 200\n",
      "04:38:27 [DEBUG] train episode 156: reward = -243.46, steps = 200\n",
      "04:40:19 [DEBUG] train episode 157: reward = -125.16, steps = 200\n",
      "04:42:12 [DEBUG] train episode 158: reward = -237.18, steps = 200\n",
      "04:44:05 [DEBUG] train episode 159: reward = -124.80, steps = 200\n",
      "04:45:57 [DEBUG] train episode 160: reward = -351.52, steps = 200\n",
      "04:47:50 [DEBUG] train episode 161: reward = -244.16, steps = 200\n",
      "04:49:43 [DEBUG] train episode 162: reward = -596.23, steps = 200\n",
      "04:51:30 [DEBUG] train episode 163: reward = -246.55, steps = 200\n",
      "04:53:16 [DEBUG] train episode 164: reward = -123.55, steps = 200\n",
      "04:55:01 [DEBUG] train episode 165: reward = -122.38, steps = 200\n",
      "04:56:42 [DEBUG] train episode 166: reward = -243.91, steps = 200\n",
      "04:58:23 [DEBUG] train episode 167: reward = -596.74, steps = 200\n",
      "05:00:08 [DEBUG] train episode 168: reward = -243.37, steps = 200\n",
      "05:01:50 [DEBUG] train episode 169: reward = -123.00, steps = 200\n",
      "05:03:34 [DEBUG] train episode 170: reward = -591.54, steps = 200\n",
      "05:05:16 [DEBUG] train episode 171: reward = -123.34, steps = 200\n",
      "05:06:57 [DEBUG] train episode 172: reward = -122.74, steps = 200\n",
      "05:08:51 [DEBUG] train episode 173: reward = -570.28, steps = 200\n",
      "05:10:37 [DEBUG] train episode 174: reward = -237.73, steps = 200\n",
      "05:12:21 [DEBUG] train episode 175: reward = -473.73, steps = 200\n",
      "05:14:03 [DEBUG] train episode 176: reward = -125.72, steps = 200\n",
      "05:15:48 [DEBUG] train episode 177: reward = -499.51, steps = 200\n",
      "05:17:33 [DEBUG] train episode 178: reward = -122.84, steps = 200\n",
      "05:19:18 [DEBUG] train episode 179: reward = -354.69, steps = 200\n",
      "05:21:00 [DEBUG] train episode 180: reward = -611.68, steps = 200\n",
      "05:22:42 [DEBUG] train episode 181: reward = -242.68, steps = 200\n",
      "05:24:24 [DEBUG] train episode 182: reward = -237.96, steps = 200\n",
      "05:26:10 [DEBUG] train episode 183: reward = -244.58, steps = 200\n",
      "05:27:55 [DEBUG] train episode 184: reward = -356.48, steps = 200\n",
      "05:29:37 [DEBUG] train episode 185: reward = -122.68, steps = 200\n",
      "05:31:18 [DEBUG] train episode 186: reward = -121.29, steps = 200\n",
      "05:33:02 [DEBUG] train episode 187: reward = -2.76, steps = 200\n",
      "05:34:45 [DEBUG] train episode 188: reward = -356.27, steps = 200\n",
      "05:36:29 [DEBUG] train episode 189: reward = -4.24, steps = 200\n",
      "05:38:12 [DEBUG] train episode 190: reward = -238.95, steps = 200\n",
      "05:39:55 [DEBUG] train episode 191: reward = -468.01, steps = 200\n",
      "05:41:38 [DEBUG] train episode 192: reward = -118.63, steps = 200\n",
      "05:43:21 [DEBUG] train episode 193: reward = -564.73, steps = 200\n",
      "05:45:05 [DEBUG] train episode 194: reward = -125.75, steps = 200\n",
      "05:46:48 [DEBUG] train episode 195: reward = -361.16, steps = 200\n",
      "05:48:30 [DEBUG] train episode 196: reward = -240.85, steps = 200\n",
      "05:50:04 [DEBUG] train episode 197: reward = -477.32, steps = 200\n",
      "05:51:37 [DEBUG] train episode 198: reward = -364.28, steps = 200\n",
      "05:53:10 [DEBUG] train episode 199: reward = -127.64, steps = 200\n",
      "05:54:44 [DEBUG] train episode 200: reward = -490.00, steps = 200\n",
      "05:56:16 [DEBUG] train episode 201: reward = -121.42, steps = 200\n",
      "05:57:43 [DEBUG] train episode 202: reward = -127.72, steps = 200\n",
      "05:59:07 [DEBUG] train episode 203: reward = -120.75, steps = 200\n",
      "06:00:24 [DEBUG] train episode 204: reward = -246.98, steps = 200\n",
      "06:01:40 [DEBUG] train episode 205: reward = -127.79, steps = 200\n",
      "06:02:57 [DEBUG] train episode 206: reward = -128.53, steps = 200\n",
      "06:04:15 [DEBUG] train episode 207: reward = -122.72, steps = 200\n",
      "06:05:31 [DEBUG] train episode 208: reward = -121.22, steps = 200\n",
      "06:06:48 [DEBUG] train episode 209: reward = -128.93, steps = 200\n",
      "06:08:05 [DEBUG] train episode 210: reward = -250.15, steps = 200\n",
      "06:08:05 [INFO] ==== test ====\n",
      "06:08:17 [DEBUG] test episode 0: reward = -121.06, steps = 200\n",
      "06:08:30 [DEBUG] test episode 1: reward = -121.41, steps = 200\n",
      "06:08:43 [DEBUG] test episode 2: reward = -4.54, steps = 200\n",
      "06:08:56 [DEBUG] test episode 3: reward = -606.56, steps = 200\n",
      "06:09:09 [DEBUG] test episode 4: reward = -126.56, steps = 200\n",
      "06:09:22 [DEBUG] test episode 5: reward = -126.88, steps = 200\n",
      "06:09:35 [DEBUG] test episode 6: reward = -119.16, steps = 200\n",
      "06:09:48 [DEBUG] test episode 7: reward = -486.39, steps = 200\n",
      "06:10:00 [DEBUG] test episode 8: reward = -501.57, steps = 200\n",
      "06:10:13 [DEBUG] test episode 9: reward = -119.34, steps = 200\n",
      "06:10:26 [DEBUG] test episode 10: reward = -126.63, steps = 200\n",
      "06:10:39 [DEBUG] test episode 11: reward = -0.48, steps = 200\n",
      "06:10:52 [DEBUG] test episode 12: reward = -485.65, steps = 200\n",
      "06:11:05 [DEBUG] test episode 13: reward = -358.65, steps = 200\n",
      "06:11:18 [DEBUG] test episode 14: reward = -125.33, steps = 200\n",
      "06:11:31 [DEBUG] test episode 15: reward = -573.09, steps = 200\n",
      "06:11:44 [DEBUG] test episode 16: reward = -239.49, steps = 200\n",
      "06:11:57 [DEBUG] test episode 17: reward = -492.63, steps = 200\n",
      "06:12:10 [DEBUG] test episode 18: reward = -128.90, steps = 200\n",
      "06:12:23 [DEBUG] test episode 19: reward = -232.10, steps = 200\n",
      "06:12:36 [DEBUG] test episode 20: reward = -362.95, steps = 200\n",
      "06:12:48 [DEBUG] test episode 21: reward = -122.09, steps = 200\n",
      "06:13:02 [DEBUG] test episode 22: reward = -125.79, steps = 200\n",
      "06:13:14 [DEBUG] test episode 23: reward = -362.22, steps = 200\n",
      "06:13:27 [DEBUG] test episode 24: reward = -584.94, steps = 200\n",
      "06:13:41 [DEBUG] test episode 25: reward = -123.96, steps = 200\n",
      "06:13:53 [DEBUG] test episode 26: reward = -127.59, steps = 200\n",
      "06:14:06 [DEBUG] test episode 27: reward = -240.20, steps = 200\n",
      "06:14:19 [DEBUG] test episode 28: reward = -232.60, steps = 200\n",
      "06:14:32 [DEBUG] test episode 29: reward = -126.65, steps = 200\n",
      "06:14:45 [DEBUG] test episode 30: reward = -555.29, steps = 200\n",
      "06:14:58 [DEBUG] test episode 31: reward = -127.88, steps = 200\n",
      "06:15:11 [DEBUG] test episode 32: reward = -242.91, steps = 200\n",
      "06:15:24 [DEBUG] test episode 33: reward = -601.64, steps = 200\n",
      "06:15:37 [DEBUG] test episode 34: reward = -126.87, steps = 200\n",
      "06:15:50 [DEBUG] test episode 35: reward = -492.83, steps = 200\n",
      "06:16:02 [DEBUG] test episode 36: reward = -471.13, steps = 200\n",
      "06:16:15 [DEBUG] test episode 37: reward = -540.78, steps = 200\n",
      "06:16:28 [DEBUG] test episode 38: reward = -127.28, steps = 200\n",
      "06:16:41 [DEBUG] test episode 39: reward = -120.67, steps = 200\n",
      "06:16:54 [DEBUG] test episode 40: reward = -358.97, steps = 200\n",
      "06:17:06 [DEBUG] test episode 41: reward = -124.68, steps = 200\n",
      "06:17:20 [DEBUG] test episode 42: reward = -506.01, steps = 200\n",
      "06:17:32 [DEBUG] test episode 43: reward = -126.74, steps = 200\n",
      "06:17:45 [DEBUG] test episode 44: reward = -350.33, steps = 200\n",
      "06:17:58 [DEBUG] test episode 45: reward = -238.96, steps = 200\n",
      "06:18:11 [DEBUG] test episode 46: reward = -122.70, steps = 200\n",
      "06:18:24 [DEBUG] test episode 47: reward = -122.26, steps = 200\n",
      "06:18:37 [DEBUG] test episode 48: reward = -358.71, steps = 200\n",
      "06:18:49 [DEBUG] test episode 49: reward = -121.65, steps = 200\n",
      "06:19:02 [DEBUG] test episode 50: reward = -478.52, steps = 200\n",
      "06:19:16 [DEBUG] test episode 51: reward = -127.19, steps = 200\n",
      "06:19:28 [DEBUG] test episode 52: reward = -0.54, steps = 200\n",
      "06:19:41 [DEBUG] test episode 53: reward = -240.66, steps = 200\n",
      "06:19:54 [DEBUG] test episode 54: reward = -356.13, steps = 200\n",
      "06:20:07 [DEBUG] test episode 55: reward = -358.22, steps = 200\n",
      "06:20:20 [DEBUG] test episode 56: reward = -119.34, steps = 200\n",
      "06:20:33 [DEBUG] test episode 57: reward = -123.71, steps = 200\n",
      "06:20:47 [DEBUG] test episode 58: reward = -122.33, steps = 200\n",
      "06:21:00 [DEBUG] test episode 59: reward = -593.70, steps = 200\n",
      "06:21:12 [DEBUG] test episode 60: reward = -0.54, steps = 200\n",
      "06:21:26 [DEBUG] test episode 61: reward = -122.09, steps = 200\n",
      "06:21:39 [DEBUG] test episode 62: reward = -121.40, steps = 200\n",
      "06:21:53 [DEBUG] test episode 63: reward = -361.02, steps = 200\n",
      "06:22:07 [DEBUG] test episode 64: reward = -122.36, steps = 200\n",
      "06:22:19 [DEBUG] test episode 65: reward = -238.50, steps = 200\n",
      "06:22:32 [DEBUG] test episode 66: reward = -245.23, steps = 200\n",
      "06:22:45 [DEBUG] test episode 67: reward = -618.04, steps = 200\n",
      "06:22:58 [DEBUG] test episode 68: reward = -122.65, steps = 200\n",
      "06:23:11 [DEBUG] test episode 69: reward = -123.31, steps = 200\n",
      "06:23:24 [DEBUG] test episode 70: reward = -246.25, steps = 200\n",
      "06:23:37 [DEBUG] test episode 71: reward = -124.21, steps = 200\n",
      "06:23:50 [DEBUG] test episode 72: reward = -237.99, steps = 200\n",
      "06:24:03 [DEBUG] test episode 73: reward = -245.20, steps = 200\n",
      "06:24:16 [DEBUG] test episode 74: reward = -125.24, steps = 200\n",
      "06:24:29 [DEBUG] test episode 75: reward = -118.42, steps = 200\n",
      "06:24:42 [DEBUG] test episode 76: reward = -615.05, steps = 200\n",
      "06:24:54 [DEBUG] test episode 77: reward = -123.28, steps = 200\n",
      "06:25:07 [DEBUG] test episode 78: reward = -128.32, steps = 200\n",
      "06:25:20 [DEBUG] test episode 79: reward = -122.68, steps = 200\n",
      "06:25:33 [DEBUG] test episode 80: reward = -351.11, steps = 200\n",
      "06:25:46 [DEBUG] test episode 81: reward = -235.15, steps = 200\n",
      "06:25:59 [DEBUG] test episode 82: reward = -4.49, steps = 200\n",
      "06:26:12 [DEBUG] test episode 83: reward = -127.92, steps = 200\n",
      "06:26:25 [DEBUG] test episode 84: reward = -232.36, steps = 200\n",
      "06:26:38 [DEBUG] test episode 85: reward = -0.42, steps = 200\n",
      "06:26:50 [DEBUG] test episode 86: reward = -358.81, steps = 200\n",
      "06:27:04 [DEBUG] test episode 87: reward = -361.15, steps = 200\n",
      "06:27:16 [DEBUG] test episode 88: reward = -235.18, steps = 200\n",
      "06:27:29 [DEBUG] test episode 89: reward = -125.39, steps = 200\n",
      "06:27:42 [DEBUG] test episode 90: reward = -353.71, steps = 200\n",
      "06:27:55 [DEBUG] test episode 91: reward = -238.33, steps = 200\n",
      "06:28:08 [DEBUG] test episode 92: reward = -121.59, steps = 200\n",
      "06:28:21 [DEBUG] test episode 93: reward = -498.14, steps = 200\n",
      "06:28:34 [DEBUG] test episode 94: reward = -244.71, steps = 200\n",
      "06:28:46 [DEBUG] test episode 95: reward = -123.82, steps = 200\n",
      "06:28:59 [DEBUG] test episode 96: reward = -1.30, steps = 200\n",
      "06:29:13 [DEBUG] test episode 97: reward = -241.89, steps = 200\n",
      "06:29:26 [DEBUG] test episode 98: reward = -127.86, steps = 200\n",
      "06:29:38 [DEBUG] test episode 99: reward = -126.27, steps = 200\n",
      "06:29:38 [INFO] average episode reward = -241.35 ± 168.17\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYYAAAD4CAYAAADo30HgAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjMuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8vihELAAAACXBIWXMAAAsTAAALEwEAmpwYAABeY0lEQVR4nO29eZgcV33v/f3V0stMz6bRNhptliXjFQusgI2xAWOwIRCzmbAEkxsSYwcSEvKEayC8l4QsFxLCG5aY1wTCcgFD8DU2AWwwm8EbyHiVbdmSbFkjjaSRNPv0Ust5/zhLnaqu7llas/T0+TzPPNNdVV11aju/81sPMcZgMBgMBoPEWuwGGAwGg2FpYQSDwWAwGGIYwWAwGAyGGEYwGAwGgyGGEQwGg8FgiOEsdgMaZeXKlWzz5s2L3QyDwWBoKu6///5jjLFVaeuaXjBs3rwZO3fuXOxmGAwGQ1NBRPtrrTOmJIPBYDDEMILBYDAYDDGMYDAYDAZDDCMYDAaDwRDDCAaDwWAwxFhygoGILiei3US0h4iuW+z2GAwGQ6uxpAQDEdkAPgfgVQDOBPBWIjpzcVtlMBgMrcWSEgwAXgBgD2NsH2OsAuBGAFcscpuWFYwx/N/fDmCy7C92UwwLzA8fGcTxifKiHf/JI+MIw+Yv8z9V8XHvvuMAgJIX4OhYaZFbdPJZaoKhH8AB7fuAWBaDiK4mop1EtHNoaGjBGrcc2H98Cu//9kO47dHDi90UwwIyWfZx7dd/i+/cP7Aoxz84UsRl/++d+OkTRxfl+CeTb/3mAN72hXsxWvTwpbuexqs//cvFbtJJZ6kJBkpZVjXEYIzdwBjbwRjbsWpVaka3oQajRQ8AMFbyFrklhoVkqhIAACbF/4XmxEQFjAGHl8Hoev/xKYSMC9ujY2Ucm6ig7C/OdZ0vlppgGACwQfu+HsChRWrLskSakCZKxpTUSpS8IPZ/oSmK444vg+ducLQIgJ9TaRmdl85SEwy/AbCNiE4hogyAtwC4dZHbtKwYl4LB+BhaiuIiCwZ53Ily82uqg6Nc6ylWgmUl8HSWVBE9xphPRO8FcDsAG8CXGGO7FrlZywqpKRjB0FoUhQmpuEimpOXUgR4a4YKh7Afqeo4vM9PskhIMAMAY+wGAHyx2O5YrE0ZjaEmUxuCHi3L85WJyKfsBjonIrmIlXFYCT2epmZIM88yE8TG0JLIDWyyNIRIMzT2yPjIahfsWvQBljwtaIxgMTY3RGFqTUmWRnc+V5TGyPiQczwAXDMVlIvCSGMHQYhgfQ2tS8hc7Kml5jKwHNcFQ8pav89kIhhZDCgST+dxaFCu8Yy4uclTSeJNHJUnHMyAEwzLRhJIYwdBijBuNoSVZMuGqTd6BDo4WkXV4t1msBMvGd5LECIYWQ8aRL7cRjqE+UYLb4kQl6SYXxpq3XtLgSAmnrGwHYBLcDE1EGDKMFr2aI8PJMl9e9kNMVXxc//O9vBDYeAkfuvkR/OTxIzg4UsTIVAUAcGikiJ/vrl3f5sQk3+6pI+P4ixsfQMUPMVH2sevQqNomCBk++9OnMDxZgReE+Nq9+3HHY0dSzVmf+clT+MxPnsKeoxNqmexIKn4IL+Ad26MHR/Hx257An33zAXz5rqdVqQ+dhw6M4DM/eQr/zy2P4t/ueKqqbMH+45P4h+8/hn//+R589NZd+Pwv9gLgnejTxyaVVlX2A9zy4EEwxnB0rIQv3/V0zeuht/nJI+OxZR+++RHc9uggAODxwTHct++4Cn38+e6jeNeXf4MrP383Do9G5ooDJ6bUvdh1aBSjRQ+MMXzyR7tx4MSU2m7f0IS6ZuMlD08eGce9+47jkz/ajccHx6I8hpOgMQQhw3/e9TSu+dr9eNeXf4OP3/YEDo4U6/5GPo9+yFKF02jRU/fn6FgJlRphtXuOTuCr9zyDr9z9TFVBvhOTFfz66RMA+Mj+/9y7H1+7d39VJBZjTLVnZKqCXz11bNpzniz7uOHOvXjk4Cj6u/PIOFbc+SwGXKNFD2+8/m48c2wSAGZUKoMxhut/vhd37TmGe/cdx0dv3VWl0T8+OIbP/pQ/yz/adXjeheuSy2Mw1KfkBXjxx3+Gf3rDOXjFmWuq1r/rK7/Bz3YPoS1j4+7rLkF3Wya2Xn/g7nj8KD5+2xPYurqAYxNlfOO+Z/GN+54FALRlbPz2I6/AV+5+Bl+662k88bFXwbbipax+++ww3nT93fjx+1+CnzxxFN998BCufelW3L7rMD77sz145KOvRNax8dihMfzLj57EivYs+nvy+Mh3HwUA/N656/Dptz5P7W+s5OGTP34SAPCVe57Bzr95Bf7ph4/j0YOj+Pofn4/33fgAHNvCZ976PPz5Nx/Asyem0FvI4HsPHcLQRBnve/lpuPzf7sTf/O4ZOG1NB6743F0AgI6sg/GyjwtO7cULTlmhjnfT/QP4wi+jTt6xCH9y0RZ85LuP4r/uH4BjEW5574UYGC7ifTc+iNPWdODuvcfxsf9+DL/73HVY1ZGteZ9+++ww3nj9PfjvP3sxzu7vAgB85/4BhAy45PQ1uOJzd6Hih9iwIo9ffuASfOXuZ/Cz3bwg5GODo1jblQMAvPNLv8YLt/Ti7644C2/497vxpy/diit3rMdnfroHXXkXf3zRFgDA/7p1F/yA4ZtXn4+rvvRrPPDsiGrLyJSHjDB/lLwAE2Uff/Af9+HvX3e2attseHxwDH/7vcfQ351HR87BL54cwhd/9TT+9c3n4jXPXYerv7oTrz13HV577jr1m6ImDMZLHvIZO7bP13/uLlx29lr85aWn4eWf/AXe/8rT8D8uPKXq2Nfd9DB27h8GAJy7oRvbN3SrdV/45T586VdP44mPXY5P/+QpfPPXvB7nyvYMXnVOn9rul08dw7u/dj/u+eAluPE3B/CJ257Arr+9vKpNOj/fPYR//MET6Mg5eNnpq7Fz/zDGij6kbJIawzPHJnH//mE8emgUU5UAv/fZX+Gnf/VSbOxti+0vCBne8cX7cO1LT0V/dx4fv+2J2PoLTu1Ff3ce1/yf+8EYlODNuza+es/+qnfnZGM0hibjxGQFxybKeODZ4dT1+45Noj1jY6oSYJ8YteiMl3xkbH7b94oR5uBoEYMjRRABn/+D8/C67eswVQkwVvQwUfbhBQxHx6uLnw2OlBAyri3IUe6h0SKePjaJih/GlsnjHBzmn/u781UF1QaFY2/jijYMT/GR8b6hSTzw7AgYY3h4YFSNxE5MVfC2F27EfR+6FN1tLsaKPkaLHvYNTWL34QmMTPER3Gff9jz85//4HQDV9vXB0RLWdGax628vw8euOAt+yDA0XsaTR8bRmXPghwwDw0VlFx+Z8jAmNJPpbPVD43yUv3eIX+OKH6Lshyh7AaYqPip+iJxr4fBoCYwxjJd8rCxkxL6jTvToeBlPH5vA4dESyn6I45NlLYAgasNYyVeFEY+OlfHCU1bgC1ftwLquHMZLXpTH4AXYf3wSDx4YwX8/PFj3HGohNb1PvOm5uO0vLsYvPvAyrGjjAjoMGX78+BE8dGAk9ht91D6eGA2Xff6sPjE4hsHRIsbLvnpOkgwMF3FmXycAYDChpRw4MYWyH8IPGSbKgRrITCU0hv0nplD0eKLaeMlDyKb3EUiN9Md/+RL8wfmbkHMtDAttGYgEg7zOZS/EgeEp+CHDs5pmp+/v7r3H8cCzI6p9V563Hn992XMAAEfGSrh//zAGhos4d0MXPviq0/HAR16BRz76SrzpvPW49aFD81q4zwiGJkN2SIOj6VUqK36IU1cXAHAzUJKJsoc1XXykKwXHwZEiDo2WsLoji8vPXosXb1sljhWqTkqPxki25dBISQmBwZGSGt3I3xzSvh8aKcIi4LQ1hSpHpBQgp65qRxAyeAFX+acqAYYmyjg8VsJ4iQuMiZKPQpYrvDnHRkmz91b8EBVhcmrPOsg6fCRYTpgnDo+VsLYzh/asg/6evGrDwZEinrO2Q52jDPWcKPuqU57OJCNfWnkN5O9KfqCu6eqOHLyAoSzMbysL/L7ITjQMGSbKPr8/4hpOlHwtgCDqzMqaWaPkBdi6uoBXnLkG3W0ZjJd8lcfAWGT++80zJ+qeQy1k9nTO5d1Hf3ce/T15TJR9TFZ8MAYkp13QO7GkPV4mjR0cKapnJ81m7wchjo6XcN6mHgDV74B8BitCAHfkHHHs+H0va/4WeS/GpvERSMEh95l3bQxPVarWy3tQCUJ13DSho28vr81rzl2Ha19yKhyLcHi0hEOjRWRsC5996/Px7pecip72DBzbUlqvnmx3sjGCocmQD3Itm27FD7G5lzvHkoLBC/iL0NfJO0GlMYyUMDhaxLpuvjzv8o5Ud66lCZmiElJFDIrR/+Bo1IkNKk1BWzdaxJrOHLrbMlV2VKkxbFlVUPuXneRv9w8jCPnIuuTxUWFHzgXAO6iSH6oXrBJEGalZx1JmlKTd+shYCWs6uclGnvu+oUkcm6jgVNGGstZ5jJc8JcymyyCW6+U1kB1ByYvKKKwWpqixkofxkq9MU3L9ZIUf6/BoCQfECHqs5KcmKZa0LNyiF6h72JFz+DXTOuahcd6hPDwwMqcoJdmxSoELAIUsP45sU5iwgRcrgdJUkx2lvEYHhyONMi2sdWiijJABp/d1IOtYsZwCvp9IMFSCUHXilcTIWve3zDSqaLzkwyJuYgWAnGsrrTTrWEqQSQFc9gL1vKUJObW9Fw0U8q4NyyKs7sji8FgJgyMlrO3KwUqYcPuEmTF5/icTIxiajJJfu6MG+EvRW8igI+tUjfKlCWCNeLD2HdNNSSWs6+KdoxwJ6qPwtIdQCY3REg6L9QPDxciEpDQF/v/wKH/Y13XnRUdS3UFYBGwW9lg9gejefXx0O172q0ZvOdcWMeX8BSt7kcYQEwxBvIM4PFpStnwpGO7fz48jBQMf4Ucag+ywptMYkgJ1XBMocp0UBOPCDLS6Ixf7rexkvYDh4YERscxT5693OFLgSMeqtJd35l2MlbyYIDsqBIMXsCqTz0xIagwAUMg5mCj5SnBWCQYvUOeb1BRlhz5ZCfDY4FjVuSW3W9eVx7rufExjCEKmTJNlP0TFD9GRddX3ZFuA2SWoTZS5hkrEO+l8xsYJoTGs6shWm5K0gUrSdKYfT8+FkNdzbVcOR8b4YE0KAR25bD7ntjCCocmQI5IjYyUEKdMkloMQGcfCuu58lfCQHc3aTv6C6maiQ9pDqGsMRdXB1TYlDQwX1Sj0oQMj8EW7DiUFxGhRPeyFnIOJcjx08dAIH8EXRIevlzW+T0SbVPwQxyb4CykFQ1YIhpLSGEI1WsvYthIMZc12X6wEGCv5SjB05lwUso6Kajl1dbs6x7LWeegvdD1k53lQXDf1Oz86J9lRSl+O0hhSkqZkuybKfmr2uhz9VoIQIePCUl6j8ZIfE2RHxyIThHTkzoY0jaEzxx38YzUEQ8kLsFIThDp6By/NW2mmHTngWNuVw9rOXCx669hEWb0P0p9Ty5QUCYZwxrWOxkqe0lABbr6UPobVHVkUvUBp5KoNXm1TkjIteqF6buU9W9vFz+2QGEQlWSsGcGnv5MnCCIYmQz5EXsBUqKOEMYaKHyJrW1jXnYvVdQE0wdAVf9gOjhS5iUk8hFnxgOoaQ5rpSr4Ejx8aUzZl3eEdmZRKavtnT0xhnYhmkfZ1iRQaMcEkOsknDo/FtgM0jcGxhMkn8jHI0VrGsVRCktQigGi0tbYzGpGt685h7xBv/5aVBdXmkmYSmKlgkO1WvgG9I5CCQfgUjoyVwBiwot2FY1HqKHa3CH3VzTUTMY2BC5xSRY7mhcaQc4XzOTr3IfHcbO5tUwJnNsjrkdU1BqEBRqakxG+8UJ1vcvZAXRt97JDUGKo7U/kc9XXl0NeViwkUfRBUCbgZpy1jw7aoyoSoT1o003kixku+et4ArjHIAZAU6JNlP6Ex1DMlCc2zEpmSckLQrunk53ZkLNJodQpZBx05R2np84ERDE2GHrGS7KyV+cS1hcYQH1HIjkRXT7vbolHQuoTGwDsbvs80U5LuaAOAbcLpDfBOZ3CkBD8IcWSshK1iXcj48TuE41h/aQZHS+jrzqtOTddY9AGo1EQiH4Mdc+pW/EhjqOVjkOcTFwxcMFoE9PfkYVsU6zz0jm9aU5IQTKNFD5Oa+aus+QJkhyLvU0fORd611b51jUCef9z57It1XMAyFnW60izRIUbyU2VfLRsaLyFjWzijr3NOdmqpMcj7JNte8kKVc5GMs9dNScmO8tBISa2TAmWsmKYxFJFzLXTlXfR152Jas649lDx+/+WgIBm9U9QKCsr7NJ3GMF7y4oJBO3dpAuT+LykYIqdyWra3Howg73cuI0xJnTlMVQL4IVPvZJKkYDzZGMHQZOgj1cFExx+ZT7gp6cRkJTVMcE1nFH9/7vpu9VlqDJGPIVSdQD1TkuR5G6N9nbdpBQ6NFnF0nDsMd4hIEgDo68orc5HeuR0aKWKdpjGUKkGqk1eODiMfgxXrwMuaYMg4lnJ66trJEaExrNFevD6hSa3tzMG1LeQcKzbC18040p9RC91sNThajAkU5XzulIKBn08h6yCXsZWAS+tQxrXIKNmZ6eclI4505zNjwPHJCnpETsvQeBkdOQd5EdY8W8qa0JXICDHZWSXNnCUvQCFroz1jVwUdHB4r4qx1nTGfRS2Noa8rDyLC2q48/JDhuNB+DmmdJI8ICpB1uBmxno9hplFJE2U/bkqKCYZIE5LPa0xjSNFGdJ9TUtCuTXkmk/R15Y1gMEToI9WkD0HvDNd184dLNyfJjqYr76roCj1BSGkMmeoR+4nJSpUgSI6aZRhhV97FaWsKGC/5ygTyfE0w9HfnURCOQdmm4SkPZT9EX1deHX+yEsRe6tUd1R0pwF+osh/GBENZuxayA9P3dViE+ukaQ7+4ZjJ0NZ+JayLclDQz57Mu0A6OlBJRKNKUFL9HHTlHCTkgMm90CgHY3eai4oeq84/MU9GxZAhlJBj4dT4xWVHJjkeFYGibq2DwAhBBCVzZdiDKLdDlAmMMRS9AzrXRIUxbOoMjvMOXGtuqjmxMuEsOj5bU/errlNeuJNZppiQ/rjEk9yO1YF3ozyQqSdcYdCGma0JKi9ban6aNSM2upD230pSkP5N93UZjMMwA2UlZVNuUlHEsFWGkCw8ZldSedVSnul2M8l2bVBy9svGLCBq5bVIQlb1QJRFlHQtn9vEs2nXdeaV9/FY4N8/p74Ijtu3rzqmXTL6Qct/ruiONQXZysl2nq8SmhClJ5jGIF1F3PmcdC0SEjB3vII6MldCRddCejV522TH1S19LIj9itOhhcobzGpT8QJmwBkeKmmCIwlVXdmTEuSdMSQnns8ypeM4a/l+af6TzXhdSMoRSdz5LVrS7ar+FnIO2jIOpyuxr/JT8UF1XiRIMorPSnc/SzJVzbRSEM1ztywtwfLKCdV05dd1PF+dbHbVWUmZQOaqWAkHXGMoiXJULBrtKY9DnpphpraNxLW8GSJiSOqsFQywqKc2UpIW3Fj2ejOfaFDs3AOo9TrK2K4djE+WapUMaZd4EAxH9MxE9QUQPE9HNRNQtlm8moiIRPSj+Pq/95jwieoSI9hDRp0l/8gwAog5pfU9blX04aUoC4uYmOcIsaIJh2+oC2jM21nRG8dKyU5H2zy2reIROcoRS9AJsEKPrtV05Nbrp784p7WPnM8OivXms6cwh41jobc+o40vzlhQMfV2Rj0FGfUjfxRl9vMM4WKUxWDGzV1lE5wBQnXMmMXI8PFqKmZHksYFIY8i5wqktfndECw+cVjB4ATauaINF/NxUHoOmgbS5/D7oprE0H8M2IRCkgDismWv0JC0gEqa5hMYAQJmSAKAjy7XGkhdW1RyajpIY/evI48hnRHcxSLNa3rVVlJREXtO1mmA4QwwA9O3CkMWcser5VomVReW3khFB0r+U9DGouSli2mBtjYFnpsejkvTyGVLzGy95UR6DH9SNStKj1EpeiJwmaGVuTdaxYj5AHSkwjsxTyOp8agw/BnA2Y+y5AJ4E8EFt3V7G2Hbxd422/HoAVwPYJv4un8f2NSVSjd+8sr3K7q+bktZ25UAJrUI+jO0ZR9n4V3fk0Nedj41M4mWFQxXT/7V79mP34XFVyK4kHIqFrIO1nTn0tmfQnrGxYUWbenF//cwJdGQddORcrOvm0SREpEaYcuQko4T6unPqpZNx4s/f1A3HIpy/pRcAfxkKWUdpKzKPQUUlBZGQkOYO2UEcHuXFAh8bHKuKEd+8sg1EUQ5Dcr/6S6g7MF/1b7+sKsRW9EK0Zx2s6czh4EhJ86VE5RVyGQsdOUdFCXFTkiYYSj7aMrbK6zh9rdCYNAE9XvYSpqRq57MkJhiEKYm3NTIXfuK2J+AHIYbGy7jlwYMAgO89dAjv/tpO9VvZ6epEPgb+vOk+BuVcFaYkPSpJ5SZ057Gptx22RZrG4OPBAyN8nuXJMvyQqXvW0+Yi41jq94dHS9i0kl+nih+qsG1pSto7NIHtf/cjHDgxpd27uP+I3x+GT//kKVz8iZ/hok/8FN+471mU/RBewBKmpGqNYaIcJRJWNHPmRNnHwPAU/vZ7u9R1UT4noZXr+8u5NrrbXKzrzqPW2FgKyPkyJ81bET3G2I+0r/cCeFO97YmoD0AnY+we8f2rAF4H4Ifz1cZmpOgFyDoW1nRk8eThePVO3Sno2ha6866yRwNRko5lEQpZB73tGWQcCx+47Dloy0SPAhEh51qqA9u2poB3X7wFX7rrady26zByroWbrn0RisLMtH1DN85c1wkiwtf/5Hys78ljRVsGb96xHmU/xCWnrwYA/NGFp6hOQY6+5GhqeJL/X9GWUechNYbT13bi4Y++UnXyfshian1W+Bh0+245CJGxo1GY7CDufHJIFQp8oVZQD+Aaw/f/7CKctkYTDH7kHPSC6s5uYLiIxwfH8NjgKF68baVaX/IC5F1L2IKLsRd/dKqibPQdOQeDohBtR87liVOaD6GQdfCWF2zEttUdKjxUN43okTAAVFSQSnDTBUO7Lhhc5MU9n6oEaM86+NkTR/HvP9+L1567Dnc+OYR/+uETeOlzVuPefcdx+64jKFYC5Xep1hj4vmSOiW5Kktcqn7GwspDBHq3qrIoO68rhueu78Dube1QY6BOHx/CBmx7Gp968XUW1rRajaSKeISzzZ4Ymynju+m48enCM53OIsO2scD7vG5rEyJSHfccmE87nuLnnwIki/vXHT2L7hm4QAR+6+RF1bp0pgsG1SY3qx0t+zPlsW9G+b3v0MP7zrmfwzgs2Y/PKdi0TPlD+F53+7jxWtMcLYOrMd/bzQlVX/SMA39K+n0JEDwAYA/A3jLFfAugHMKBtMyCWVUFEV4NrFti4ceO8NHipUvJCNfJKRnfoDldAOmWjTkN/APu78+oFfOVZa6uOw2vBeOrzn756K/7wws343kOH8I8/eAJ7jk6g5IVYWbDxhat2qN/pzuxPvOnc2D71CpftWd6OKMLGQ9614Wid+YnJ6PhScEmHaZojUAoy6fjTR7UZx0IliITHda86HZeesbrqvM9c1xnbb9JUIykmtIhklFLZC9DTnsHKQhaPHhxVIY0AH9XnHFtoTrxTIQLaXDvuYyhzh2dnzsXLTl+NRwaiUuYr2jM4MVnBhCgRou9bXjMgaUqKPnfkHLSJbbifIav8DRNlX11LPUR3cLSILasK6RpDLt6V6KYk2fnmXW6yHJooIwwZLIuU1tvXlUNbxsGOzStUyfZHDo6CMW4ek23Q73t7xkGxEsAP+Ii+R/hQ5HlkXRGV5IXRuWmCtKj5paRgmPL4/6sv3oLnb+zB+f/0E5Udrp+jvL4511bRT2NasULd/zZVCTAgS32U4hFlJWH20p3ZAC9QmLzGOn3deZzR1wnXnh+jT0OCgYjuAFDdqwAfZozdIrb5MAAfwNfFukEAGxljx4noPADfJaKzAKTpTKnGT8bYDQBuAIAdO3Y076wfc4CPRLmtdqLsIwiZegArCcEgR0sSvbP8W1FNtBa8FkzcXt3XlcfrtvfjH3/whBqpJkc6M0W+TNLHMFGOoj5si5BxrCjCRrPnduScasEgojmk41VGtGR0wWDzDkK+uFddsCmmJaVeA4fXwyklbNRAdTHD5DZFL0CfY6O/O48fPXYkdp2GpyrqnOR5SE0uL7QUQDqJo85c75jWdua4YCj7scS95D3Tr1NHzoVtEYKQxUxJMjJpUjm9vVgHJs19h0ZK2LKqkK4xZOO28DSNIevaWNORhRcwDE9V0FvI4vBoSUTJRe3sFOf8uCiPoTvsdadvLmPHOvfOPP+dDD3N2Nz5PKLNTzKuhZROivdHLpfH4tfPwqqOLByL8ITQzPVzzGeiwRdfJzPMRVkWP4iVqVdzZSQmyqr4ISYrftX1PGtd/XLohayDH77vorrbNEJDgoExdmm99UT0TgCvAfByJjJeGGNlAGXx+X4i2gvgNHANYb328/UADjXSvuVIyZcaA791kxVfvUh6fSD+346ZGbwgVJEP03WKevXIZE0cQNhThblkrnRqjkgZKRM7/mS8kwP4C3EE5ViHKdfrGkM5IRiyLtcYkqGB9Uj6GKLllupcIo0h4eD0QuQzNvq6cqj4IfYfn1IayMiUh5wjfQCuuBauaKettI+JkqccqnxbPYIqh8cGx6pmREs6n/OurYRBW4ZrJFII55OCQcuPkB0lrw8lBIMwW6RpDDnXgmORGmzogkE6ZKXGwK9bGb2FbGo9IHktnhjkHTLP6JbmKE0wyMlyxLqufGTSAaJQ5bIXn5tZChk5kLAtUsJEPR/iuvV15yLBkDIYqSpWqJmSHDsS2E9qmevyukpGpryYwFsKzGdU0uUA/ieA32OMTWnLVxGRLT5vAXcy72OMDQIYJ6LzRTTSVQBuma/2NSvFCvcxSBu7ngSl1wcCRFRNQmPI1FFPdXKurez++kMrOxpZ43+uGgPAO3nZ/vFEAlHejYqU6ceX26SZkuSLLsNVswmNoSL8EFnHqqpYmUZWMyXJTgfgDns5spQ23qrsWi9Azo2iw/TM35FiBbkUjUGeq+4Q1X0p+mcZQaU7PIHIVyOvie7oz7u25pR2VahuNIKOOk/VgcU0Bn6uaRoDEcUEu6bExEb70kdwZFxW3C1VCQa5HymQ9GJ3sWcxExfc8h7J/A8ZlVTxQ0yJbY5PVlSOhRxIrCxkVBkV3VEOcJPrMRUcoA1GMknB4Kp3AoiXZQGiwoUTQpDrGvJo0WvoPZoP5jMq6bMAOgD8OBGWejGAh4noIQDfAXANY0wWbLkWwH8A2ANgL4zjuYqyHyUKAfGQvmpTkh3LwOUaw0wFg6XMElntoSUi1aFLs9ZckYX0+Hl4MedePhOVNU6akoB0R+BIkbdXljzOJH0MfohSJag7U5eO9NGU/SA2W9sqUTQNiBLlqjUGnnmrF0GTtYJGhI9BPx/VeWcsVSV1IqFF8aCCeKz7RMmL+RhGpiqwaiSf5Vw7ZmKS907a3+X/8VKkJeiZ1jL0OU1j0I8DxEtiROYZW2XdHx3TBEOiUJxtEdq1e6QXU0wOUnSh0ZlL0xhE8qO4P9JZDUTalfT/TJT8KAvZkYIhmnktrSSGLuB1/0VZ+A703/C2ceERhEw9U8NTlSofw2Izn1FJW2ssvwnATTXW7QRw9ny1aTkgO+PIpBOF/smy0sqU5FoxlbUSsBlrDPmMrWzOyc6/kHUwJpye2QYEQ0c2yoKdKPmxjM9cogOQdCqNQTclVWsMZS3BDAAyjo3RqQofyc/AjATwzkGG7K4sZLDnKHcSr2jPqPmWpSkp6aAuC1NSvy4YtAxZ2ebOhAaUd201SdF4QmOQQnl4ylPXaqLsxxxxk5UA7Rk7nnyWdQEUkc9EgqGQnamPIXI+S1NSyQ9S73tBHAeoEZXk2uoaHBkro+QFOCGS25J05FzVHt1clE08F/qcCm0ZGxnbigsGoTXLNgxN6IIh0hj4ufoxHwMQZcPzNqUIBmUSdPDMsSktwY37GFYVsrHB20Q50sZWFbLYNzTZchqDYR4oiQiGKHM4euikdlBLY6j4wcw1Bq3zTAtNlOp1oxqD7mOIv3iW9rlaY9A7TNlWuS9ZtVUfNUtHvLT9z4Sca6nOaZUYVcoONZqkSPgYND9EEDJUghA5h8ejy05GZnADqHY+awUBAT5Knij7Mc2Ib8+3W1HIKOd9UijVCiXNuVbM9NGmhasCkY9hQiv6l2ZKKnthqnDV758e16AXics6NnraXBwZK2lltKuze/V9lbyoNIr+LEh/TFEbwGQdS7U363BBUfYDdY5HtVwUqWGnlbSQ90cmOwKJZ076cDLR9RwveXGNwQ/QW4iHnOr+G3lcmRW+lDCCocmQkUBp1UmT2b58tKQ7n1mss6xHLpM+Ygf4SytV8kZU4I5sZEri9vT0zFJZdVIeW/8PIHX0OlH2Y/MF6OGqM30J9e3kqLIj66iQ0orP52AG4pnQkQOTh97qNYDUvmuakvjy45NlMFYdBio7p46sw6+f3plpIZQ6MlonL8JhAW6Ki5zP/B7Enc/884nJinquBsX81GU/iJXcluiO8lhJjITdfk1nDkfGykoDSdMYOjWfjpzMRi8bIc9HNyXJ8FSZKyM1hoqmMcgBjd7JR4Ih6tiTpiQZSq0fW/8vNTmZ6yLrdemDAUBONOXHjsuvzdLqipdWawzTIju2ZHVSID1ctZTwMczY+RzTGKqzXE+mxhCEcWecvt+kvVwKjzRTks54yY9HJdnRnA0zjaRKK5TWkXNVdvLR8ZKK108TDGrUmSYY5EgzW21KAiJbeCERBlpQGoajfDRlj5vNomkn4+en7zurOZ9V5rMYTU9ppqQod4GPsDeuaMNUJcBo0ZuhxqCZkhImydWdORwdL8Um3qm1L6KomGPejZvI8hlZVTfSJjLaNJtZzccg23BcRLrppSaieSL8mD8EiDSGpK9ADlZ0QSuFT3vG5v4sL64x9LZnYkJXfx5aJirJMD/oCW5AvA6LSnCz9QS3eFSSPuKqRz6TbsoBeKdyPCWUdLbIXIy05CU93DKtWFut8gSSKsGghavOSWNo5y9xQYy0y16o/At85KrNDpcYdcqom1WFOhpDNi4YZAZxUmOQ28l6V9LhmXOsKvOGRPoxchlbMyU5cG1eklxG7Mg5po+OR7OhyagrmQ1+aKQkfAzVXYfe1jARleRYpMyYazqyODpW1ibeSTMl8TZvXNGmnM/J+5Z3+WQ58h3IZ4QpqRwPVwWiaqZSXunlQVZqGoPSPsTv5L1LCoZq53MkaGQVWxnN5ghn+rrufEzoxp4HIxgMjVAWYZBtrg2iWuGqeoJbNJKtzCIqKWnL1SmIGv9AYw90IesiCJkaHadpDMlOLtmRJtsgK7iOl7ya4aozHZ3po+J2YUIqiP+VIFTZrJtXtsd8DFJIyM4z1ZSkjdz1//JcjopwTv084+fPpyKVPgbuWLaq2g1EZpmcw53P+qT2+UyUaT0lwlX1Glyy8z5NFPI7MDwFL2A1NAaRi+FY8TwGL4xdc5n9PDBcRHebm+rz6cg5sAg4ZWU7DxmuBLHBin6t9HwbrjEIU5IdCQYZmCBJ0xik9qWHM+dcGysL2VjHrx87KeABxEKbcw7X7td25VTkUtLHoO9vqbBQJTEMJwkZQ25ZhELGiU00XhH1gfSHOu58nl0egyTNxxBtN/exhRxhylFpmo8h+cLIl2llzF4ftaErz7WZZIKbLKJXrAQx/0k99FGxdPjrYZ7PHOORSaesbMNvnonmTi4lbP6nr+2AaxM2rIhCH+W6dd05tGVsVQtInq8yJSV9DOJ7e5YnOcoRvB6Kmuxof/93NuCUlW3IOJYSblILa8vYyrcg/+tTxkqtaJvQGPYfn6y6NqptQoh15t1YSYyiF49iWtOZRRAy7Do0WnMimivOXYc1HTnsGZrAs8enUqPJIsEQ5dtkHEs5vnNubcGgawy68zlNozxtTSHW2QOAa/MqwTL8Vhcc+rYZhz8367rzyLs29k9M1fAxGMFgmCOyJkwy21KS7PizwuEq69J4QThz57Nm40+an/RRbCO2URlxI0feaRpDW6KTu3jbKnznmgvUCFZvK8A7JWnmysauhbT7hjPXGLTtcq6Nqy/egi2r2nFQtPfpYxPIuZZIeEtzPvPfX3bWWtz1Py/BykJW5VPIdd1tGTzy0ctU+QTZqR9N0aIAPifDttUFOLYlKpWOC1NSZCZKK8j2+ufxogJv/p31OHt9VG4hn7ExJfImJhNzM1gUFQ7cuIKXXpfaRC5lgCEd9CvaMgg0yeAHcROmjPB6eGAUrzxzTdV+AOCFW3rxwi29+Ov/eijyMSSeBTVvh2bWjAUc2NF3mcwm0etGFXIOMo6FSal9Ja7fZ9/2fKTlQ/7wfRehq6064VIXDFnHwlXnb8barhx+vnsIE2UfI1MeLEqakpaW8cYIhibh0n/9Bd68g7/c8iEq5JyYKSkZuy9firLPTQ1ewGaR4JZu4wfSw/bmgjSx7BYlBwo1fAw6lkXYsTleFTUmGLR96EJQjiQntLmPpyMuGCz88UVbAAD/tfMAAOCpoxNY39OmMnAlSQcmEamM37xrC8EQtUGvqSPPd+8Qr63T1xkfUb/jgs14xwWbAfAR59B4GUURwpyrIRh0ztu0Audtiq5fmzAllf0QIYMqnwFEk9IDvCPlE9Dz72mRYFds78em3nZ87md7YkERIQMs7Rl64Skr8Lvn9OHMdZ144/PXV+1HJ5+JchWqfAyZyJREJDKdE/dcvg/JgpPdmsYgo/wmyr7KWNepVeV0tZZ3oz+7upkq69r4wwt5oc/79w9jrOTh0GgRaztzaEsJuV4qLC0xZUiFMYY9Rydw7z6eIK5PwpKMStJfDPmASz/DbExJtUaf8rhquxmaZdLoTwiGZOZzreMn0cMY9TBHvfPKaE7ImWoMMT+L9uLKtu0dmsD6Hm4i8AIGX5unIvl7ifID1GiD/M2Th8fRlXfViDSN1WIKzKNjJWR1U9IsRp9tLp/FTZqRVmvmDb1URSHnYEV7Rpn90oRrzrVx/pZeEFEsjyFkDJa2eU97Bp97+/PxnpdtTY1I0pGhwcWUkbxsw/BkVK02qTFnE99VG9rifoB2IRjmWhhSf3b1e6YfUwZbDAwXsa47H9O6lpopyQiGJkAWJnv6GLfvytFFIevEopKqTUmRxgDM0vmcqd2BxUb2DYx01nTmRPVKXkUz5mOoYS+vhWyHLhhio0fxmbGZm79ybvqLK39f8kKs78mr7WSVz6jeTnrnmdxfbL247pMVPgNcPWSnuv/4VKwO0mw6mbYs73hlqOqa2HzDkbbSkXXR05ZRU2hm69x3m+IlMULGYM9xMkYZWTdV9lMEQ6QxyOckWWpd94XICXWAaG4KaSptzzrclKSZ+WZDLR+Dfp06RNDGnqMT6O/Jw7GjEidJx/pis7RaY0hFzpgmyzBI52khl+J8ThkhlYQNeS55DGmdWywiqIEH2rYIa7tyynmYGpU0w5dUagedubjjL1pvVW07HUlTUrJtAJ9iNRIUQex/WgeTrMqZRF8+nWCQnbg0f8z2mgHC+SyyrIH4RPQy8cy2+MRNK9ozyjFdzxxnEcVmcAtCFjMlzQbZ4Y8Uvdo+hqmK+pwMOJAFJYGoJlLGttCeiepHEZEqm12qVJuSZoL+7HbnI9OTLqjkwOfEZEWZUZPRTUsFIxiaAOkAlJqDVEE7Ez6GZEVR+dCV/RB+yMAYkJlhHkOujiknnlzW2AMtzUl6CKV+/NmO7pOOP0kmJWt12n3WKAuiRzVxjYF/16f7TP4m+duaGoO2fMN0gkGb/EePSprNPcm7jtAYhGDokjOkRYJHRjGtaM+oaKN6GkPSlMQYZlTNNg35rA9PVheai3wMnhL8sl0W8dBlfUAgnb3ZFH9Me9bGZMVPrRw7E2TVYSDpY6jO2gei514eq5GaY/OBEQxNgBek18IpZONRSVVzEIjPZS9U+5htraS0TjRtJqu5sr6Hd356CKW+35mGlspr0pnXnM+J0aPa9yxqJSX3r7dNtj8SwNFcwsnfq/2o2fXS74NrWyoXYzqNQTeN5Bx7Vn4ZCZ8Rz1clt5UwyDjKLCc7NN0JW19jiJuSuMYw4ybFkOfkh6zqvOQzGoSsSmPIOLwciT44kNdLN7upkhY5F5PlQEV4zRa9vHlXDXNmIUUwzEXLWwiMYGgCfG2uYSDufC56gXJ6Jp3PWWX7DuD5fB+zqa6qH0tHn2mt0akFo5IDcSfrbF+YZLVSAKn+ltnsUx/F1XIUSuczEE3vWVdjmMF5yXXTCYaca6tOKJ+xVRtn5XwWU6VGGkOU4a1nWQPx2P+6PgaLYgluIZu7KalePo0u4OU6KQjke6A/A9KxrleZle9IIWur6qpzDaiQ16mrRgCE7qCWz33kF1paXfHSao0hlaTGoBfuAqIJVpIagxrJeiHKoiT3bOZj0PehU0iUb2iE9d3ptWjyiVo00yFHebUcf3ET2+yugWNRagG1nMuTnOQ1khPmJEtA6KgOrM55SS1pU299wQBAJVhlXWtaM1Ua+Qx37o4VfbG/qAREclKkFe3161NJrLSopLn6GOoIhpjQSDifs0pQVPsY8rrZTaxvz3Dnc1q46kyRU6e2az64eFRSdP3WJTQGE5VkmDWVKlNSlMcARHVgkj4GZUryA+WnmGmCm975JZETxpyMUU7NImWzjUpSpqR0jSFNYE5HxrZUfLyOvDbre9pAREqIRT6G2pEtSRNGGtJenZzZLA3ZkeccOzL/zWLEK52wcp6CtUowuFHBvqwUDJHpqr6PAQgTzmd7rj6GlM5ffU8JJ84kNAb93nW1ueK5tatG6gVRBG8qUZV3NsiseP29iDufI41CfpYCzJiSDLOmlilJqqYyoiQZlaRGsl5YVXl1OuqZcrg91T0poxxpay0kagLNdiQVmZJqJ7gl9z0dRKTqC8WOJQSBbLvsSGSYKndgpl/nXB2Bq7evvzsf01JqIUfBOXduPgb5GxltpGsM8p7IuSJ0jSGtJIaEawy6KQkN+xiA6vNybVL7ldslBYJ+39syUa2rpPNZad+zmOEvSWfOQc61E+bcaF9y8KPP6rdUNQaT+dwEJE1JkV1UznErBIMfpppPuMYwO+dzdpoHtpB1Zixk6tHXHY1QdbryLizipYpnggpXrRGVlJ2D8xlALKNYkrEtWMT9C/r+VLhqpXZky0wih3oLmRkLL+kT4O2cQx6DEgwVFRkmO9DkpEg9iWzhWliElAS3xk1JyWMSEfIuD7eVfhX5buil5/V9deTcmI8hKRiAuYeOrunMYWWhGBMG+vHbMw6I4rPCyXuWNlXqYjJvgoGIPgrgTwAMiUUfYoz9QKz7IIB3AQgA/Dlj7Hax/DwAXwaQB/ADAO9jenhDi1LLxyBfXFlwrcr5rCW4zVljqNGJFrLOnM0DOlnHxulrO7A5YU/vLWTxvT97cawmUj10W7Es65BMdlLbzqLjzLl21eiYiPDXl52OC7f2xvZXimkM9QVDvY7/U7+/fcbXdo1WamOueQwAcGy8LDouwpZV7diysr2qxLkelVSvI7NSnM+NJLhJ0s5LTkErt0tqDMmM9ZefsRr9Wtax7JjbY2Ve5tZJf+Cy0zFZ8WsOSCwx1eepqwqx89OruS4V5ltj+BRj7F/0BUR0JoC3ADgLwDoAdxDRaYyxAMD1AK4GcC+4YLgcwA/nuY1LHukf6BBlluVLsHV1AX1dOfz9fz+GHZt6UkxJIirJC5SfYqbzMbg2YV1XrmZkzNquXCyJqRG++54LUzWZs9Z1pWydjm4zztgWimFizuc55DHw/dmpI8hrX3pq1f6KlQA/efwI9hydqNm5zMR0oGcfT4duStq2pgP93fkqIVuPvOZjkJ3jTde+CI7FtaIz+zpx1rpOADziS0Yc1fNVWURVPoZGE9z455Tw34QwzCR8DckZ3/7Xa89S36W/AYhrDHM1JXW18RImjDGlNSUHYv91zQUxAaubtZYSi2FKugLAjYyxMoCniWgPgBcQ0TMAOhlj9wAAEX0VwOtgBIPSGDb2tuHJI+OqE23POviPd+7AlZ+/Bx+55dG6JTG8xFwN00FEuPMDL6s5cv3fbzwHOEm63Ml4MXTTQNa1UPQSBQXrmCTqkXWsaUeQcv1DA6P46PceAxHwjvM3pW77mnP7eJZyAzWmdNZ1R0lop64q4K7rLpnV76Xg3zc0gc0reQVVfZT9g/ddpD5bFqGnjdfnShZW1Kk2JSFWK2k2TFdPKGkSSmoKMpdBFpKM7zvqlE9WmRd5zIyYPTH5vm3qbY99v3LHBpzVP/MB0EIx34LhvUR0FYCdAP6KMTYMoB9cI5AMiGWe+JxcXgURXQ2uWWDjxo3z0OylhRQMV1+8pepBO2tdF1555hrs3D+cUl010hhUVNIsbJn1nJ+rO2Y+ql0IuvOuKpoWmROiFzymMcyiU+7Mu1Wlv5PIjuTxwTEAwM1/eiG2b+hO3fbUVQWc+pJC6rq5cE5/Fz73tufj4tNWzen3p6xsx0XbVuKXTx1TEUr16GnLqAz8WlQ5n0MGZ4429LRchdj6hMlTT3CTSMHQ5sbP732XblP3ST/3eo71mZJ1bDCGugIUAM7b1IPzNvU0fLyTTUOCgYjuALA2ZdWHwc1CHwMfV34MwCcB/BGAtCvF6iyvXsjYDQBuAIAdO3Ysex+E7NS3rCzgnPXVo4v1PW343sODCMK4im9ZhIzNX4rKLPMYmo2rLtiMi05bFauwWTNcdRad1D++/mzY0wx3LYuPSvcN8SKHpyRGhfMJEeF3n9vX0D7+6MWn4JdPHZtWAAK8+NxYyau7TVp11TmHq6ZUtdVRprnEPY9HpNkA/Kq6XrKMOpAwJZ0EDTbrWGhm92hDgoExdulMtiOiLwD4b/F1AMAGbfV6AIfE8vUpy1semdns1PAP9Pfklb0/OdrJOhbKXoiKyHxeroKhq83F9rZuAOmhilnN5jyTMFDJ1tUzdH67NkaLHjpyTt1S2UuRl2xbhTP6OlX4bT1WFbI4Nl6uu41tJUpizGDkXAspdMt++gRLqqZWVYJb/N5bVN+MmjYXSCNkHOtkWVoXhfmMSupjjA2Kr68H8Kj4fCuAbxDRv4I7n7cB+DVjLCCicSI6H8B9AK4C8Jn5al8zUZkm1FSGTQLVD3/WtVDyI+fzyQgxXeooB2RK2e35cvTlhWDY0DNzx+9SwbII/3XNBapGUz3+8hWn4YSYMa3m/ohiM7gxxjDDmIdUZHZ2erXa+H1N0xiyroW2jFNXOLVn5+aDqsVSCz+dLfPpY/gEEW0HNwc9A+DdAMAY20VE3wbwGAAfwHtERBIAXIsoXPWHMI5nAJEpqVZEkT7SSz6QWYfP+zxb53Mzk0kZNVpiMp/5EgzSAT1dfaOlSjLBsBZybup6nMyoJECak7zUe5dM6kvTFjN2dS5KkqzDE9Mqwcynfp1uf83MvAkGxtg76qz7BwD/kLJ8J4Cz56tNzYo/jcagZ1ImNYKsa8UT3JylFS89HyQLqUkytnVSXvo0ZMezYcX05pjlDhGgm9eDcO4JbkDU+dd1PidqI8USPV17RhPhtGdtVKbCaaPQZsLJcGAvJs3d+hZhuqzlnGtjlagcmRQMOcfmJTGC1tEYagmGrJYEdrKJBENzagwnEzsRlcQaKIkB8GurT9+aXAdMH5WUjEhKQ+ZxnBQfg201tdnWlMRoAqYzJQHczzA0Xo7NWAVEGoPMfHab+GGdKRmbh6wmR6kZ25rx/A6zRQqcZvQxnGx45nP0PWggKgngZcTzrp3qI0hWSY3yGKLnvKfNnZFgigrbNf6OvO2FG9U714wYwdAEzKTOUX93Hg88O5LiY+ARHbOtrtrMZN300VrGsWYVqjobpPnBmJK4KSk4SfMxAPH5E6rWKY0hEa6q3ee/u+LsaXMvgJNbTv6K7akpWE2DEQxNgDdNuCoQzYRWZUpybQxPViKNoQUEQy01PuOcvIzjJHK/643GAIsoFq4angTnc80SI5l4VFI2oTkAMy8xcjJNSc2OEQxNgDIl1Um0kvMaVDmflcYQwrbopBS+W+r0dedjk9pLXn32WqyaRR2i2dCVd9HXlTOdCtJLYjTy3G3sbVMlzZOs6+Yz6HWLyq89bS4++KrT8epzZp/0V8g5J2VWwuWAEQxNgBeEcCyqG9lx+lqeiLWykI0tzzq2KIlRXbdlufKXl56G97xsa9Xy97/yOfN2zPe9/DT8QY36SK1G0vkchAwNKAz48KvPQC1L0KvP7sOFp65UZiAiwrtfcmr6xtNQyDhLbsKcxcIIhiXMickKcq4FP2R1zUgA8DubV+Cu6y6pyl7NuVxjKPvhjCurNjsZZ+EjQtZ25bB2BjOutQJEBMZ4YhsJs9Jcy24D9Wt2WRahZ4ZzdkzHZWevQXeTZa3PF0YwLGHeesO9eNHWXjA2M99AWkmDrGMrU1Izh88ZmgfpTwgZYAtHdCM+hoXiktPX4JLT1yx2M5YEpqdYooQhw96hCRwdL8MLwjnbPbOOxedj8FvHlGRYXKTFU5qTeNntpS8YDBGmp1iiHJssww8ZSpUAfsDmbAbKuZHG0Ao5DIbFRwoBJRhC1lCCm2HhMT3FEuXIKK9gWRLlLOaqMeQzNoKQYaLsm2gLw4IgzUbS/9xogpth4TE9xRLl8FgJAFDyQnghm3On3iscc4dGSsaUZFgQpAyQpeAbzWMwLDymp1iiHBGCoVgJ4DUQUbS6k4evDgxPGVOSYUGInM+aj8EIhqbC9BRLFCkYpCnJmeOkuXIKzrGSj6zRGAwLACnns/zPYB695sLcriXK4VEuGMrSlDTH0f7qjijhrRVKbhsWH+lPkGUxGp2PwbDwGMGwRJE+hqLHTUmZOZqSegtZZfM1zmfDQiCFgPQxMBOu2nSYnmKJokxJXgA/nLspybYIvaJMhnE+GxYCK2FK4glui9cew+wxPcUSRZqSSl6ASjB3UxLAJ3AHWmMuBsPiQxQ3JYUNlsQwLDzz1lMQ0beI6EHx9wwRPSiWbyaiorbu89pvziOiR4hoDxF9murN3r2MKVYCjJV8FLIOQgZMlX24DQy5ZGSS0RgMC4FtRSUxGGNgDKmT7BiWLvM55/Pvy89E9EkAo9rqvYyx7Sk/ux7A1QDuBfADAJcD+OF8tXGpIv0Lm3rbsOvQGMZLjSWnSQe0EQyGhUAviSHNSSbBrbmY955CjPrfDOCb02zXB6CTMXYP4zroVwG8br7bV4+DI0UcGiku+HGlGWlzbzsAYLzkNWQGkiGrJirJsBCQ5nyWDmgjF5qLhRhCXgTgCGPsKW3ZKUT0ABH9goguEsv6AQxo2wyIZVUQ0dVEtJOIdg4NDc1PqwFcd9PD+Mh3H523/dfi2AQvh7FeTBM5WQlOiinJRCUZFgK9JIZMcjNRSc1FQ6YkIroDwNqUVR9mjN0iPr8VcW1hEMBGxthxIjoPwHeJ6CwAaU9O6vQcjLEbANwAADt27Jh+Mtc5Ml7yq+ZQXggmyz6AyGkMNNapK1OScT4bFoC4KYm/nsb53Fw0JBgYY5fWW09EDoA3ADhP+00ZQFl8vp+I9gI4DVxDWK/9fD2AQ420r1F0VXghmRCCQZ+NbbqJeuqxyvgYDAuIrVVXjUxJRjA0E/PdU1wK4AnGmDIREdEqIrLF5y0AtgHYxxgbBDBOROcLv8RVAG5J2+lC4YcMAVs8wdBbiGamakxj4D4GIxgMCwFptZLkuMqYkpqL+Z7B7S2odjpfDODviMgHEAC4hjF2Qqy7FsCXAeTBo5EWNSIpCEOE4cI/0JNlHznXQlsmuj2NmIFWd2axoj2D/p7qGd4MhpONnuAWGudzUzKvgoEx9ocpy24CcFON7XcCOHs+2zQbFk9jCFDIxicmdxp4s7KOjXs+eInRGAwLghXTGISPwUiGpsLM+VyHMGQIwoU/7mTZR3vWQc6NOvJGI4qyjj39RgbDSUAJhhBqYGUS3JoLM4Ssgx8yBOHCS4bJMs96zmeiznyu8zEYDAuNHpUkFW4TldRcGMFQh8WMSmrPOsg5umAwt8rQHOimJJPg1pyY3qYOfhhFVSwkkxWuMeRcIxgMzYcsBByaBLemxfQ2dVgsjWGyHKA968SS64wpydAsxMJVhSXW5DE0F0Yw1MEPwkUzJRWyNiyLlHAwGoOhWbC1stuBikpazBYZZou5XXVYPI3BR7vIYZAOaMe8WYYmIZrBTTMlGY2hqTC9TR0WI48hDBmmKtyUBEA5oI0pydAsxGolmZIYTYkRDHUIwujBXigmK7wcRkEKBpHLYJLTDM1CWkkMk+DWXJjepg6LoTFMlgMAiDQG15iSDM2FFAKMwYSrNimmt6mB1BRq+Rj2DU3gxl8/e9KPKwvotWe5QJCCwZiSDM1CWtltY0pqLoxgqIEvBEItU9J37h/Ah25+RE14DgAHTkzFvs8FOReDNCXllWAwt8rQHOgzuBnB0JyY3qYGUlPwawiGohcg1FTlp49N4uJ//hnufOpYQ8edVBpD3MdgBIOhWZAaA5/BjX82PobmwvQ2NfBFZo4c8Xz3gYPY+cwJtb7sh2I7vv6Rg6NgDBhscI7oiXLS+WxMSYbmIq0khlEYmgsjGGoQJHwM//Kj3fjavfvV+pLHncQVUX51z5FxAHw60EZICgZjSjI0G9EMbjBlt5sU09vUQPkYmMjgDBn8IDIrlT0uEDyhOTx1dAIAMF5uTDAkTUlZIxgMTQal5DGY6qrNheltaqBHI4WMCwpPm5yh7HONwRPCQgmGktfQcSdEuGpSY2hkzmeDYSGJ5mNgZj6GJsUIhhroTmc/5DWT9GUlqTEEISp+iGeOTQJo3JQ0WfZhUeR0NgluhmYj8jEgmo/BmJKaioZ6GyK6koh2EVFIRDsS6z5IRHuIaDcRXaYtP4+IHhHrPk1iKEFEWSL6llh+HxFtbqRtjRJoZqMw5BqErjFIH4MXhNh/fFIJjYmT4GNozzpqhJUzpiRDkyEfVTMfQ/PSaG/zKIA3ALhTX0hEZwJ4C4CzAFwO4N+JSE4ucD2AqwFsE3+Xi+XvAjDMGNsK4FMAPt5g2xrC12ZuC9J8DL7UGJgyI7VlbIyXZ29K+tVTx/DdBw4CiGZvkxhTkqHZoJQ5n818DM1FQ4KBMfY4Y2x3yqorANzIGCszxp4GsAfAC4ioD0AnY+wexjPBvgrgddpvviI+fwfAy2kRDZO6j4GbkcKYsNA1hqeOTIAIOKe/a04awxd/tQ//fDu/jJMVXzmeAeDF21biTeetR09bZq6nYjAsKFaaYDA+hqZivuwT/QAOaN8HxLJ+8Tm5PPYbxpgPYBRAb9rOiehqItpJRDuHhoZOctM5eo2kUJTf9lI0hkoQYmB4Cms6cljVkZ2Tj2Gk6OHIWAlhyDBRDmKC4Yy+TvzLlecaG62haVAlMUKoiXpMVFJz4Uy3ARHdAWBtyqoPM8ZuqfWzlGWszvJ6v6leyNgNAG4AgB07dsxLlTvdbOQLx3OaxuAHDGU/RD5joyPnYmwOgmF0yoMfMhybKGO85KEjO+1tMRiWLLEENxWVtJgtMsyWaTUGxtiljLGzU/5qCQWAawIbtO/rARwSy9enLI/9hogcAF0ATmAeCUOGl/7zz3DzAwNV65KmJMbiwkI3JVX8EBnbQkfOwcQcfAwjRf6bwdESDg4X0deVm/U+DIalgqVVV2Umwa0pmS9T0q0A3iIijU4BdzL/mjE2CGCciM4X/oOrANyi/ead4vObAPyUNVqRbhq8MMQzx6ewb2iyap0emlrxo9BUiW5KqgQhXIfQkXVQ8sLYdtPBGMOoEAz7jk3g6HgZm3rb5nQ+BsNSQK+uGpg5n5uSRsNVX09EAwAuAPB9IrodABhjuwB8G8BjAG4D8B7GWCB+di2A/wB3SO8F8EOx/IsAeoloD4D3A7iukbbNBKkBVFI6cl1jqATCbCSWMcaiqCQ/rjEAs8tlmCj76li/fporSBt722d7KgbDkkFN7WnmfG5aGjJmM8ZuBnBzjXX/AOAfUpbvBHB2yvISgCsbac9skR2951crJro/QRXME4JEfgd4uGrFD5FxLBRyLgCey7CifWZRRCNTkenpvn1cMGxaYTQGQ/MSlcSITElGY2guWlqO+0G1iUgS1DElyTpJABcg5SBExrGVxjA2i7IY0owEAPtE9rQxJRmaGSkEWCzBzQiGZqK1BYPUGFIEg+5jSJbYLvmBWlfxQ3jSlCSiiSZmUUhPagyyrHZnzkG3yVkwNDG2VivJzMfQnLS0YJACIdXHoEUgeUFtjcELGCpBiKxjoUOYktJ8DHfvOYYDJ6aqlo8UKwCAras7AACbjH/B0OREPoZoBkSjMDQXLS0YpM9AT1xT61JMSXJ7XWNQ4aqO7nyuNiX9+Y0P4IY791UtlxrDGWu5YNhozEiGJodEr8K0zGejMTQXrS0YlPO5WmMIWYpgEA5pmcMARILBtQmFXG1TUrESqHwFHeljeI4QDMbxbGh20hLcjI+huWhxwVDb+RzTGJQpicVCVeWySpDUGKoFgxeyVE1iZKqCnGth80puQjKOZ0OzY2tlt+VrZARDc9HagqFuHoMWrqr5FIKQpWoMGdtG1rGRcayqqCTGeMnusRSNYWTKQ3c+g3P6u9Dfncd5m1Y0fF4Gw2KSNoObsSQ1Fy1dlCfpVNbRy1/ogsMPmZqkR/5W+hgAoCPrVFVYlSU10jSJkaKH7jYX67rzuOu6Sxo7IYNhCWDFopKMj6EZaWmNIQgj53PJC/DG6+/GQwdGYuuAyMfAtw3VtJ5ynTQlAUBHzqkSANIslSYYRoseuvLuyTkhg2EJYGkJboGKSjKCoZloacEgo5EqfoijY2Xcv38YjxwcBZDuYwC4JqFrDFMVLiSySjC4Vc5n+fs0H8PoFNcYDIblQtp8DEZjaC5aWjDozmepBUjtoKbGEIYxH8NkhQsBOSdzIetUCQAZ9TRZCWL7BXgeQ3feJLQZlg+yuqrufDbzMTQXrS0YNOdzOVH2Ip75HMR+I7fNuRYmhXYgTUntWQeT5Wj75L6S/ocRozEYliEWRRNcASbBrdlobcGglcRICgY9KknXGPwgikoqZF0lBFyhMbRlbBS9uGDQf69HLJW8AGU/RJcRDIZlhkWEkDEzH0OT0tqCQUYl+Ux13hW/WmPQM6O9kAsRIqA9a0emJCcSDJPldOczEBcMMuvZOJ8Nyw0uGGDmY2hSWloweDGNQfgYhBAIUuZ3BoQpyQuQdSxk7GpTUj5jo1iJawx6OKwemSSFSsFM5WlYZlhWvCSGURiai5YWDL5WRK/KlJRSEkOuL3kBcq4N17aUKSmjmZKmvACsxu91wSAFSM61T+p5GQyLjTQlhYyByISrNhstLhgijSFpSgpqhauG3PmcdSy4NqnQ1KwyJTkIQlb1G4kesSR9EW0ZIxgMywuLCEHIQ1ZNRFLz0dqCQUtwqxuV5OlRSUmNIWFKEqN/3ZxUy5Qkt8kbjcGwzCCK5nw2/oXmo9E5n68kol1EFBLRDm35K4jofiJ6RPy/RFv3cyLaTUQPir/VYnmWiL5FRHuI6D4i2txI22aCzGMIQqZG79NpDJ5IcMs5XDBIAaI7n4Eo8Q2IV29N0xiMKcmw3LCIwERUktXSw8/mpFGv56MA3gDg/0ssPwbgtYyxQ0R0NoDbAfRr698u5n7WeReAYcbYViJ6C4CPA/j9BttXFz3aSI78pRDwUybqAcRUnn6ArGvBdaInXvoY8mmCIWZKqtYYjCnJsNywLVIlMYwpqfloSJYzxh5njO1OWf4AY+yQ+LoLQI6IstPs7goAXxGfvwPg5TTPHitf6/Bl4tnM8hi4xpCxo+ZFeQxc1hZraAxjumAQGkPeCAbDMsMiHsARMGZMSU3IQih5bwTwAGOsrC37T2FG+ojW+fcDOAAAjDEfwCiA3rQdEtHVRLSTiHYODQ3NuWGxjGSpMaTkMaQV0cu6FhxNR642Jfmx30j0PAbjYzAsV0iZkqISGYbmYVrBQER3ENGjKX9XzOC3Z4GbhN6tLX47Y+wcABeJv3fIzVN2UT3nJgDG2A2MsR2MsR2rVq2arhk10c1FUjBI81IQMrhCI4jlMYiy21nHjpmSsk7ClORVm5I6svHKq8bHYFiu8JIY/D0ycqH5mNbHwBi7dC47JqL1AG4GcBVjbK+2v4Pi/zgRfQPACwB8FcAAgA0ABojIAdAF4MRcjj1T/LDalKRrDBnbghcE1XkMfoCcaynBAVRrDGmmpJ72TNz5XAlgUSRUDIblgq3lMZhyGM3HvPRIRNQN4PsAPsgYu0tb7hDRSvHZBfAacAc2ANwK4J3i85sA/JTpWWLzgJeiMVSCKCopK0byybLbZaExSIczEDmf24WPQXc+SwG0oj1TpTHkXdsk/xiWHSRKYvAEN/N8NxuNhqu+nogGAFwA4PtEdLtY9V4AWwF8JBGWmgVwOxE9DOBBAAcBfEH85osAeoloD4D3A7iukbbNBN35PF5OOp+Z6uxjzmcRlcQ1hmofQz7FxyDLbPQmNQYvMI5nw7LEsuTUnqbkdjPSULgqY+xmcHNRcvnfA/j7Gj87r8a+SgCubKQ9s0V3ME8mnM9ByFRnn5bHkHVsMM0FMpM8hp6kxlAxgsGwPJElMXhU0mK3xjBbWtq4neZjiDKfo+k6y55uSuKZz/mMpTQKIsART3/OqRYM8ji97RlMVQJ1jGIlMBFJhmWJpZmSTFRS89HagiEtwc1PMSVpGkPRC+GHDG0ZR5mSXNtSdlTLIuRdG8VYuCo/zvqePADg8GhJ7MsIBsPyxCJpSjJ5DM1ISwuGmPO5Ip3PfJkvwlWJ4uUxZB5CXtRKAoCsHb+MbRk7pjFIYbNlVQEAcODEFAAuGEyoqmE5YhEhDBlCZibpaUZaWjDo2c0y/kl3PtsWKRORZKwoBEPGhuvwdZlEuGlyTgY/DOFYhI0r2gAAz0rBUAlMOQzDssT4GJqblhYMeg0jicpjCBgcy1JqsMxZkCUt2jI2XJH5nBQMSY3BCxhc20JfVw62RTgwHGkMxvlsWI7w6qowpqQmpaWnDvODELZFMVNRUmOQajAvfxFGGoNrK2FRrTE4scznih/CsQmObaG/O49nTxQBcI3BmJIMyxHbIjGDmzElNSMtrTH4Aasy5fghd5j5IRcaMgbbsQiOZSkfQ1vGUSUxMkkfQ8L57Ieh2mbDinzMx2BMSYbliD7ns0lwaz5aWjB4YbVgAHgUktQYZKidbRMcm+I+BnuGpiSfqW03rmiLBIMJVzUsUywRtMEYg93SvUxz0tK3LAhDVSZbxwt4SKqjmZJsIri2FfMxZGoIhqTz2Qu4KQkA1ve04fhkBRNl34SrGpYtFHM+G42h2WhpH4MXsFjHbAmHWcUPq3wM8v9oUZqSbNXZu9OEq3paToSMTNpzdAIAkDOmJMMyxCIe6RcyM7VnM9LSGoMfhLGooEKWy0kvYAhCBseO+xhc21IluHVTUrI6alvGwaSe4OaHMVMSADx5eJxvazQGwzKEz+AmE9wWuzWG2dLagiHhY+jIuQC46YdrDFakMQgfg6Qt40SmpBSNoSqPQfx2gxAMu49wwWDCVQ3LETJlt5ua1hYMAUPWsSE1XakxlP3IxyAnaXMsK5bspmc+pzmf/ZCpnIhKEDmfe9pcFLIOnhSCwYSrGpYj8Yl6jGBoNlpbMIQhXJtUp13ISVOS5mOgyMegm47499p5DEA0WQ83JfFtiQjre/JKMBjns2E5YmkagxEMzUdrC4aAwbGjKqlSY6j4oSpjYVlaHoPo3KX5x6ljSgKAKY/7GbgAirbZuKINR8bKYtuW9v8blinKx2AS3JqSlhYMnuj85Wg+TWOQ5iOLSGQ/Rw7jWuGqyTkZdFMSEDmgASCfaelbYFimkEpwYzAKQ/PR0sNVXg8pMhF1SI1By2OQarBjRwJEagw1i+i58XmfdVMSEDmgAeNjMCxPZNltgtEYmpHWFgwhNyW5KaakIEhEJVmaxiDMP24NU5KMbhqeqojjVJuSJMaUZFiOSB+D/GxoLhqd8/lKItpFRCER7dCWbyaiojbf8+e1decR0SNEtIeIPk2ikAoRZYnoW2L5fUS0uZG2zQQ/4CN5OeKPTElMCA29iF61j6GWKWnLqnYAwL6hSbU/RxMMG1bk1WfjfDYsR2RUUhgawdCMNGrgfhTAGwDcmbJuL2Nsu/i7Rlt+PYCrAWwTf5eL5e8CMMwY2wrgUwA+3mDbpsUPuB8h6Xz2ghCBiL+2UqKS2pTzWZiSEhrD6o4sOnKOym6uJExJ63s0H4MRDIZlSDwqabFbY5gtDQkGxtjjjLHdM92eiPoAdDLG7mGMMQBfBfA6sfoKAF8Rn78D4OU0z2UZPWHikb6CjpxmSgoZbIqcz3oegxQMtfIYiAhbVxfw1FEekqpXVwW4X2FNZ5Z/Ns5nwzLEIhIlMUyCWzMyn73SKUT0ABH9goguEsv6AQxo2wyIZXLdAQBgjPkARgH0pu2YiK4mop1EtHNoaGjODQzCuPO5PZsQDFq4qqVtl3f5doWsg+42N+YzkGxbXcCeo7opKf5ybOhpi2krBsNywrKAgDGT4NakTOv5JKI7AKxNWfVhxtgtNX42CGAjY+w4EZ0H4LtEdBaAtCdEzpJTb118IWM3ALgBAHbs2JG6zXQwxpTtP+l8LopJdpzkfAx2XGPIuTbu/5tXpKrKW1cX8O2dAxiZqsALwqpCextXtOGJw+OmVr1hWSJLYoBBDa4MzcO0goExdulsd8oYKwMoi8/3E9FeAKeBawjrtU3XAzgkPg8A2ABggIgcAF0ATsz22DNFztrmaqN2GU0kBYNtU42opMgvUEtN3rq6AIBXUfWCsEoz+MMLN2PH5hUn63QMhiWFrZuSjFxoOubFjkFEq4jIFp+3gDuZ9zHGBgGME9H5wn9wFQCpddwK4J3i85sA/FT4IeYFXwgGW8tPkD4GmX+QzHyW280k92Db6g4AUjBUm5Keu74bb3vhxpNwJgbD0kPmMRhTUnPSaLjq64loAMAFAL5PRLeLVRcDeJiIHgJ3JF/DGJOj/2sB/AeAPQD2AvihWP5FAL1EtAfA+wFc10jbpsNXGkNtU5JtWWq0Y6eYkurR351HzrXw5JEJBCGrMiUZDMsZi0jM4GZMSc1IQ9lVjLGbAdycsvwmADfV+M1OAGenLC8BuLKR9swGP+CVTx2b1NzN7ZlqjcEW5iOnhimpFpZF2NzbriKTjGAwtBIkTEmBmY+hKWnZ3soLuMagF9HLuvyzrHHEZ3Dj21uWXhJjZvJ0dWcOh0aKABDLYzAYljvSlGTCVZuTlhUMfsg1Blcvn21byDgWSnpUUizzeeYaAwCsKmQxOFrixzEag6GFiKqrMhN514S0bG/lC41BZjS7NimtIPIx6JnPFlwrXhJjOlZ2ZJT24RjBYGghZHXVkEGFfBuah5btrZTz2bZQyDkqVNW1rcjHUFUrKV52ezpWFbLqc8aYkgwtBK+VxIyPoUlp2dKeuvP56ou24LXPXQeAl7eIRyVpeQwqKmlml21VRyQYpOPaYGgFYrWSjGRoOlpWMCjns2Wht5BFrxjdZ3SNIeFjcEXnPmNTkqYxuI4RDIbWgfsYuNZg8hiaj5btrZTzOWHiiWsMicznWeQxAHGNwZiSDK0EqagkM1FPM9LCgiFyPuu4tiYYKMp8ti3C9g3duHBrb0wTqIe+nTElGVoJi4j7GJjRGJqRljUlyaikZBipaxNOTPKZ13ra3VgRvedt7MHX//j8GR+jO+/CsQh+yIwpydBS8DwGaUpa7NYYZkvL9lbK+WxVm5KCkKGnzcW567s1U9LsL5VlEXoLGQAmwc3QWujOZ2NKaj5aVjB4YZT5rCM1iJefsQaObSk1OFkEb6ZIP4NJcDO0EpZKcINJcGtCWra3qqkxiA78lWeu4evtyMcwF6SfwQgGQythUVTa3iS4NR8t21v5SmOoNiXlXRsXn7YKQDSReVKAzBSZ5DbX3xsMzYglMp/558Vti2H2GOdzYiT/zhdtxivPWqPmXFBF9OY46lkpTEnJeaENhuXMBm26W5Pg1ny0bG8l8xiSI/nzt/Ti9c+LJpmzG/UxGFOSoQX5vXPXoaeNl5kx4arNR8v2Vl4NjSGJnscwF160tRcvOrUXfV25Of3eYGhGcq6NPzh/E4BI6zY0Dy17ywKhMUzX4dsN+hhOX9uJb/zJ+TOaDtRgWE684/xN6Mg5WNNpBkXNRsv6GKKJeqYRDPbc8xgMhlZmdWcOv/nwpWZQ1IQ0OufzlUS0i4hCItqhLX87ET2o/YVEtF2s+zkR7dbWrRbLs0T0LSLaQ0T3EdHmRto2HTJc1Z2mw29UYzAYWhkjFJqTRofBjwJ4A4A79YWMsa8zxrYzxrYDeAeAZxhjD2qbvF2uZ4wdFcveBWCYMbYVwKcAfLzBttWlVrhqEmlqMpEVBoOhVWhIMDDGHmeM7Z5ms7cC+OYMdncFgK+Iz98B8HKax5RJfaKeejSax2AwGAzNxkIYzn8f1YLhP4UZ6SNa598P4AAAMMZ8AKMAetN2SERXE9FOIto5NDQ0p0Zd85JTse8fX43sNPkFdoNRSQaDwdBsTCsYiOgOIno05e+KGfz2hQCmGGOPaovfzhg7B8BF4u8dcvOUXbC0/TLGbmCM7WCM7Vi1atV0zaiJZdG0dVz0iXoMBoOhFZg2KokxdmkD+38LEtoCY+yg+D9ORN8A8AIAXwUwAGADgAEicgB0ATjRwLFPCkZjMBgMrca8mZKIyAJwJYAbtWUOEa0Un10ArwF3YAPArQDeKT6/CcBPGWOpGsNCEkUlmXBVg8HQGjSUx0BErwfwGQCrAHyfiB5kjF0mVl8MYIAxtk/7SRbA7UIo2ADuAPAFse6LAL5GRHvANYW3NNK2k4WlopIWuSEGg8GwQDQkGBhjNwO4uca6nwM4P7FsEsB5NbYvgWsYSwoZtGQ0BoPB0CqY3m4aZLiq8TEYDIZWwQiGaZCagolKMhgMrYIRDNNgopIMBkOrYQTDNFxwai+uecmpeM7ajsVuisFgMCwILVtddaZ05V1c96rTF7sZBoPBsGAYjcFgMBgMMYxgMBgMBkMMIxgMBoPBEMMIBoPBYDDEMILBYDAYDDGMYDAYDAZDDCMYDAaDwRDDCAaDwWAwxKAlMOVBQxDREID9c/z5SgDHTmJzliPmGtXHXJ/pMdeoPot1fTYxxlKnwGx6wdAIRLSTMbZjsduxlDHXqD7m+kyPuUb1WYrXx5iSDAaDwRDDCAaDwWAwxGh1wXDDYjegCTDXqD7m+kyPuUb1WXLXp6V9DAaDwWCoptU1BoPBYDAkMILBYDAYDDFaVjAQ0eVEtJuI9hDRdYvdnqUAET1DRI8Q0YNEtFMsW0FEPyaip8T/nsVu50JCRF8ioqNE9Ki2rOY1IaIPimdqNxFdtjitXjhqXJ+PEtFB8Rw9SESv1ta12vXZQEQ/I6LHiWgXEb1PLF/Sz1BLCgYisgF8DsCrAJwJ4K1EdObitmrJ8DLG2HYtrvo6AD9hjG0D8BPxvZX4MoDLE8tSr4l4ht4C4Czxm38Xz9py5suovj4A8CnxHG1njP0AaNnr4wP4K8bYGQDOB/AecR2W9DPUkoIBwAsA7GGM7WOMVQDcCOCKRW7TUuUKAF8Rn78C4HWL15SFhzF2J4ATicW1rskVAG5kjJUZY08D2AP+rC1balyfWrTi9RlkjP1WfB4H8DiAfizxZ6hVBUM/gAPa9wGxrNVhAH5ERPcT0dVi2RrG2CDAH3IAqxetdUuHWtfEPFcR7yWih4WpSZpJWvr6ENFmAM8DcB+W+DPUqoKBUpaZuF3gQsbY88FNbO8hoosXu0FNhnmuONcDOBXAdgCDAD4plrfs9SGiAoCbAPwFY2ys3qYpyxb8GrWqYBgAsEH7vh7AoUVqy5KBMXZI/D8K4GZwFfYIEfUBgPh/dPFauGSodU3McwWAMXaEMRYwxkIAX0BkCmnJ60NELrhQ+Dpj7P+KxUv6GWpVwfAbANuI6BQiyoA7e25d5DYtKkTUTkQd8jOAVwJ4FPy6vFNs9k4AtyxOC5cUta7JrQDeQkRZIjoFwDYAv16E9i0qssMTvB78OQJa8PoQEQH4IoDHGWP/qq1a0s+Qs9AHXAowxnwiei+A2wHYAL7EGNu1yM1abNYAuJk/x3AAfIMxdhsR/QbAt4noXQCeBXDlIrZxwSGibwJ4KYCVRDQA4H8B+N9IuSaMsV1E9G0Aj4FHo7yHMRYsSsMXiBrX56VEtB3cBPIMgHcDrXl9AFwI4B0AHiGiB8WyD2GJP0OmJIbBYDAYYrSqKclgMBgMNTCCwWAwGAwxjGAwGAwGQwwjGAwGg8EQwwgGg8FgMMQwgsFgMBgMMYxgMBgMBkOM/x8m0LAKR2H0XAAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "def play_episode(env, agent, max_episode_steps=None, mode=None, render=False):\n",
    "    observation, reward, done = env.reset(), 0., False\n",
    "    agent.reset(mode=mode)\n",
    "    episode_reward, elapsed_steps = 0., 0\n",
    "    while True:\n",
    "        action = agent.step(observation, reward, done)\n",
    "        if render:\n",
    "            env.render()\n",
    "        if done:\n",
    "            break\n",
    "        observation, reward, done, _ = env.step(action)\n",
    "        episode_reward += reward\n",
    "        elapsed_steps += 1\n",
    "        if max_episode_steps and elapsed_steps >= max_episode_steps:\n",
    "            break\n",
    "    agent.close()\n",
    "    return episode_reward, elapsed_steps\n",
    "\n",
    "\n",
    "logging.info('==== train ====')\n",
    "episode_rewards = []\n",
    "for episode in itertools.count():\n",
    "    episode_reward, elapsed_steps = play_episode(env.unwrapped, agent,\n",
    "            max_episode_steps=env._max_episode_steps, mode='train')\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('train episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "    if np.mean(episode_rewards[-10:]) > -150:\n",
    "        break\n",
    "plt.plot(episode_rewards)\n",
    "\n",
    "\n",
    "logging.info('==== test ====')\n",
    "episode_rewards = []\n",
    "for episode in range(100):\n",
    "    episode_reward, elapsed_steps = play_episode(env, agent)\n",
    "    episode_rewards.append(episode_reward)\n",
    "    logging.debug('test episode %d: reward = %.2f, steps = %d',\n",
    "            episode, episode_reward, elapsed_steps)\n",
    "logging.info('average episode reward = %.2f ± %.2f',\n",
    "        np.mean(episode_rewards), np.std(episode_rewards))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "env.close()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
